fix: admin init --force preserves keys, credentials and CA (#536)

Expand the publish workflow to build the two Ubuntu NVIDIA GPU variants
alongside the Alpine eks-node/ecs-agent AMIs. The image build+publish
scripts are already manifest-driven and Ubuntu-aware; only the workflow
lacked the GPU legs.

Refactor the single-job body into a matrix over the four image manifests,
resolved by a new 'variant' input (all|cpu|gpu) so the slow 16G GPU legs
can be skipped when only the CPU AMIs need republishing. GPU images do not
fit a RAM-backed /tmp, so set SYSTEM_IMAGE_BUILD_DIR onto the runner's
on-disk _temp; legs run serially (max-parallel 1) on the single ci runner.

.github/workflows/e2e-ddil.yml

@@ -208,3 +208,12 @@ jobs:
           quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
           ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-ddil-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 30

.github/workflows/e2e-nightly.yml

@@ -348,6 +348,15 @@ jobs:
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
   # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
   # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
   # matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected

.github/workflows/e2e-suites.yml

@@ -267,6 +267,15 @@ jobs:
             "$VM":/tmp/start-net-capture.sh
           ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
   single_suite:
     name: Single env · suite
     needs: single_provision
@@ -467,6 +476,15 @@ jobs:
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-td-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
   # ---------------------------------------------------------------------------
   # Multinode env
   # ---------------------------------------------------------------------------
@@ -617,6 +635,15 @@ jobs:
             ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true
           done
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
   multi_suite:
     name: Multi env · suite
     needs: multi_provision
@@ -789,6 +816,15 @@ jobs:
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-td-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
   e2e_suites_status:
     name: E2E suite blocks · summary
     if: always()

.github/workflows/e2e.yml

@@ -466,6 +466,15 @@ jobs:
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
       - name: Fail job if any suite failed
         if: always()
         run: |
@@ -881,6 +890,15 @@ jobs:
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
       - name: Fail job if any suite failed
         if: always()
         run: |

.github/workflows/gpu-e2e.yml

@@ -0,0 +1,369 @@
+# GPU E2E — drives wattle's persistent, shared-host Spinifex stack over SSH to
+# exercise GPU passthrough (raw EC2, ECS task GPU exposure, EKS pod GPU
+# exposure). See docs/development/feature/wattle-deploy-and-ci.md (Part B).
+#
+# ---------------------------------------------------------------------------
+# ONE-TIME HOST-SIDE SETUP — NOT part of this repo, a human must do this:
+#
+#   1. Register ONE self-hosted GitHub Actions runner on a banksia host with
+#      the dedicated label `ci-gpu` (in addition to `self-hosted`). Do
+#      NOT add it to the `ci-single` / `ci-multi` pool and do NOT give any
+#      other runner the `ci-gpu` label — a second labelled runner would
+#      let two GPU jobs race the single RTX A1000. With exactly one runner
+#      carrying the label, `runs-on: [self-hosted, ci-gpu]` alone forces
+#      serialization; the `concurrency:` block below queues same-group runs
+#      when that runner is already busy with a live job.
+#   2. Drop the wattle SSH private key on that runner's filesystem and wire
+#      it into this repo as `secrets.WATTLE_SSH_KEY` (raw private key
+#      contents). Optionally override `vars.WATTLE_SSH_HOST` /
+#      `vars.WATTLE_SSH_USER` if they differ from the defaults below.
+#   3. Confirm the runner user can drive wattle's control plane over SSH. The
+#      job calls `spx` (read-only checks) and the `aws` CLI with the already-
+#      configured `spinifex` profile directly — no wrapper scripts, no install
+#      on wattle (tf-user is already in the `spinifex` group there):
+#        ssh tf-user@192.168.1.13 'AWS_PROFILE=spinifex aws ec2 describe-regions'
+#   4. Manual-work mutex: an agent doing hands-on GPU testing on wattle can
+#      block CI by touching a sentinel file in the RUNNER's home directory
+#      (NOT on wattle):
+#        touch ~/.ci-gpu-paused     # blocks CI
+#        rm ~/.ci-gpu-paused        # resumes CI
+#      or by setting the repo/environment variable `CI_GPU_PAUSED=true`
+#      (no runner filesystem access needed). Either one causes this workflow
+#      to skip cleanly (job succeeds, all work steps skipped) instead of
+#      queuing behind manual work indefinitely.
+#
+#   Runbook: docs/development/feature/wattle-deploy-and-ci.md (Part B).
+# ---------------------------------------------------------------------------
+
+name: GPU E2E
+
+on:
+  workflow_dispatch:
+    inputs:
+      test_run:
+        description: "Optional -test.run regex scoped to tests/e2e/gpu (empty = run all GPU tests)"
+        required: false
+        default: ""
+  push:
+    branches:
+      - main
+      - dev
+    paths: &gpu_paths
+      - "spinifex/gpu/**"
+      - "spinifex/instancetypes/**"
+      - "spinifex/daemon/gpu_adapter.go"
+      - "spinifex/daemon/gpu_pool_reconciler.go"
+      - "spinifex/handlers/ecs/gpu_report.go"
+      - "spinifex/handlers/eks/nodegroup.go"
+      - "spinifex/handlers/eks/nodegroup_userdata.go"
+      - "spinifex/vm/vm.go"
+      - "spinifex/vm/lifecycle.go"
+      - "cmd/ecs-agent/gpu_discovery.go"
+      - "cmd/ecs-agent/gpu_ledger.go"
+      - "cmd/spinifex/cmd/admin_gpu.go"
+      - "tests/e2e/gpu/**"
+      - "scripts/images/eks-node-gpu/**"
+      - "scripts/images/ecs-agent-gpu/**"
+      - ".github/workflows/gpu-e2e.yml"
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths: *gpu_paths
+
+permissions:
+  contents: read
+
+# Single dedicated runner (see setup note 1 above) + this group together
+# enforce single-flight on the one RTX A1000: a second dispatch has nowhere
+# else to land (unique label) and, if it somehow overlaps in scheduling,
+# queues here rather than cancelling/stomping the in-flight run.
+concurrency:
+  group: ci-gpu
+  cancel-in-progress: false
+
+env:
+  # OTLP export to the central Elastic sink (wattle is the sink itself — see
+  # Part A.2). Repo variable overrides; carried for correlation even though
+  # the GPU suite's own daemon-side spans are emitted by wattle's already-
+  # configured obs-agent, not by this job.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
+  WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
+  WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
+  TEST_RUN: ${{ inputs.test_run || '' }}
+
+jobs:
+  gpu_e2e:
+    name: GPU E2E (wattle)
+    runs-on: [self-hosted, ci-gpu]
+    timeout-minutes: 75
+    env:
+      ARTIFACT_DIR: /tmp/spinifex-gpu-e2e-artifacts-${{ github.run_id }}
+    steps:
+      # Manual-work mutex gate — see setup note 4 above. Checked BEFORE any
+      # SSH to wattle so a paused run touches the box zero times.
+      - name: Manual-work mutex gate
+        id: gate
+        env:
+          REPO_VAR_PAUSED: ${{ vars.CI_GPU_PAUSED }}
+        run: |
+          set -euo pipefail
+          PAUSED=false
+          if [ -f "$HOME/.ci-gpu-paused" ]; then
+            PAUSED=true
+            echo "::warning::ci-gpu runner paused via ~/.ci-gpu-paused sentinel on the runner — skipping this run (manual GPU work in progress on wattle). Remove the file to resume CI."
+          fi
+          if [ "$(echo "${REPO_VAR_PAUSED:-}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+            PAUSED=true
+            echo "::warning::ci-gpu paused via repo/environment variable CI_GPU_PAUSED=true — skipping this run."
+          fi
+          echo "paused=${PAUSED}" >> "$GITHUB_OUTPUT"
+
+      - name: Paused — nothing to do
+        if: steps.gate.outputs.paused == 'true'
+        run: echo "ci-gpu is paused; job exits cleanly without touching wattle. See gate step above for the reason."
+
+      - name: Clean Workspace
+        if: steps.gate.outputs.paused != 'true'
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/checkout@v7
+        if: steps.gate.outputs.paused != 'true'
+
+      - uses: actions/setup-go@v6
+        if: steps.gate.outputs.paused != 'true'
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Install wattle SSH key
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          install -m 0600 /dev/null "${RUNNER_TEMP}/wattle-ci"
+          echo "${{ secrets.WATTLE_SSH_KEY }}" > "${RUNNER_TEMP}/wattle-ci"
+          echo "ssh_opts=-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${RUNNER_TEMP}/wattle-ci" >> "$GITHUB_ENV"
+
+      # Healthcheck: confirm wattle's persistent AWS-compat control plane is
+      # actually up before spending time building/shipping the test binary.
+      - name: Healthcheck — describe-regions
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" \
+            'AWS_PROFILE=spinifex aws ec2 describe-regions --output text'
+
+      # Ensure the base EC2 GPU AMI is present — the raw-EC2 GPU passthrough
+      # test is the always-on baseline, so CI self-provisions it rather than
+      # depending on a manual import. `--name` selects the catalog entry (pulls
+      # the prebuilt image from iso.mulgadc.com); `--ami-name` is REQUIRED to
+      # name the AMI ubuntu-26.04-nvidia-gpu-x86_64 — without it the import
+      # falls back to the generic ami-<distro>-<version>-<arch>, which the
+      # tests/e2e/gpu filter would never match. Idempotent — skipped once
+      # wattle's persistent loop-fs already has the named AMI (the 4.4G source
+      # qcow2 is cached in the images dir, so a re-import wouldn't re-download).
+      # `spx admin images import` writes local spinifex storage, so it needs
+      # the `spinifex` group active (sg); aws-CLI calls hit the gateway over
+      # HTTPS and don't.
+      - name: Ensure base GPU AMI present (import-if-absent)
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          REMOTE='
+            set -e
+            BASE=ubuntu-26.04-nvidia-gpu-x86_64
+            present() {
+              AWS_PROFILE=spinifex aws ec2 describe-images \
+                --filters "Name=name,Values=$BASE" "Name=state,Values=available" \
+                --query "Images[0].ImageId" --output text 2>/dev/null
+            }
+            ID=$(present)
+            if [ -n "$ID" ] && [ "$ID" != "None" ]; then
+              echo "base GPU AMI already present: $BASE = $ID"; exit 0
+            fi
+            echo "base GPU AMI $BASE absent — importing prebuilt from catalog..."
+            sg spinifex -c "AWS_PROFILE=spinifex spx admin images import --name $BASE --ami-name $BASE"
+            for i in 1 2 3 4 5 6; do
+              ID=$(present)
+              [ -n "$ID" ] && [ "$ID" != "None" ] && break
+              sleep 5
+            done
+            [ -n "$ID" ] && [ "$ID" != "None" ] || { echo "import ran but $BASE still not available" >&2; exit 1; }
+            echo "imported base GPU AMI: $BASE = $ID"
+          '
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
+
+      # Preflight — the base GPU AMI is REQUIRED: the setup step above just
+      # ensured it, so a miss here means the import silently failed. Hard-fail
+      # rather than run a false-green all-skipped suite. ECS/EKS node AMIs are
+      # opt-in tiers (heavier — node AMIs + cluster spin-up); their tests
+      # self-skip when absent, so only warn.
+      - name: Preflight — verify GPU AMIs
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          REMOTE='
+            set -e
+            present() {
+              AWS_PROFILE=spinifex aws ec2 describe-images \
+                --filters "Name=name,Values=$1" "Name=state,Values=available" \
+                --query "Images[0].ImageId" --output text 2>/dev/null
+            }
+            BASE=ubuntu-26.04-nvidia-gpu-x86_64
+            ID=$(present "$BASE")
+            if [ -z "$ID" ] || [ "$ID" = "None" ]; then
+              echo "required base GPU AMI $BASE missing — cannot run the baseline EC2 GPU test" >&2
+              exit 1
+            fi
+            echo "required: $BASE = $ID"
+            for OPT in spinifex-ecs-node-gpu spinifex-eks-node spinifex-eks-node-gpu; do
+              OID=$(present "$OPT")
+              if [ -z "$OID" ] || [ "$OID" = "None" ]; then
+                echo "::warning::$OPT not imported — its GPU test(s) will skip (opt-in tier)"
+              else
+                echo "opt-in: $OPT = $OID"
+              fi
+            done
+          '
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
+
+      # Same build command CI's other e2e workflows use to cross-compile a
+      # suite into a shippable binary (tests/e2e/Makefile e2e-build target,
+      # SUITES_BUILD scoped to just gpu). Building on the runner keeps go
+      # toolchain / module cache off wattle entirely.
+      - name: Build GPU e2e test binary on runner
+        if: steps.gate.outputs.paused != 'true'
+        working-directory: tests/e2e
+        run: |
+          set -euo pipefail
+          make e2e-build SUITES_BUILD=gpu
+          ls -la _bin/gpu.test
+
+      - name: Install go-junit-report on runner
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          go install github.com/jstemmer/go-junit-report/v2@latest
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Ship GPU e2e test binary to wattle
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          ssh ${{ env.ssh_opts }} "$VM" "mkdir -p \$HOME/spinifex-gpu-e2e-bin"
+          scp ${{ env.ssh_opts }} tests/e2e/_bin/gpu.test "$VM":spinifex-gpu-e2e-bin/gpu.test
+
+      # Same invocation shape e2e.yml uses to run a pre-built suite binary
+      # (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
+      # -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
+      # net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
+      # the advertised gateway (mulga-7suds), so the override is required
+      # here (proper fix tracked separately: tests/e2e/harness/env.go should
+      # prefer spinifex.toml's advertised bridge).
+      - name: Run GPU e2e suite on wattle
+        id: e2e
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          if [ -n "${TEST_RUN}" ]; then
+            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          else
+            TEST_RUN_FLAG=""
+          fi
+          ssh -tt ${{ env.ssh_opts }} "$VM" \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-gpu-e2e-bin; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               SPINIFEX_WAN_IP='${WATTLE_SSH_HOST}' \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./gpu.test -test.v -test.timeout=60m ${TEST_RUN_FLAG} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-gpu.log'; \
+             exit \${PIPESTATUS[0]}"
+          E2E_RC=$?
+          echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
+          exit 0
+
+      - name: Collect artifacts from wattle
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          ssh ${{ env.ssh_opts }} "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+          echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
+
+      - name: Convert test log to JUnit XML
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log)
+            SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Publish test summary
+        if: always() && steps.gate.outputs.paused != 'true'
+        uses: ./.github/actions/junit-report
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          title: "GPU E2E results"
+
+      - name: Analyze e2e failures
+        if: always() && steps.gate.outputs.paused != 'true'
+        uses: ./.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "GPU E2E failure analysis"
+
+      - uses: actions/upload-artifact@v7
+        if: always() && steps.gate.outputs.paused != 'true'
+        with:
+          name: gpu-e2e-artifacts-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      # Cleanup is NEVER a host teardown: it never runs setup-ovn.sh,
+      # dev-reset, or cluster-reset. It only reconciles EC2-API-level state —
+      # terminate any GPU-typed instance still pending/running (the Go
+      # suite's own t.Cleanup already does this on normal pass/fail exit;
+      # this is the backstop for a hard `-test.timeout` kill or a workflow
+      # cancellation, which skip deferred Go cleanups). Termination triggers
+      # gpuManager.Release() on wattle, returning the A1000 to the pool for
+      # the next queued run or manual test.
+      - name: Cleanup — terminate leftover GPU instances, release GPU
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          REMOTE='
+            set +e
+            GPU_TYPES=$(AWS_PROFILE=spinifex aws ec2 describe-instance-types \
+              --query "InstanceTypes[?GpuInfo!=null].InstanceType" --output text | tr "\t" ",")
+            if [ -z "$GPU_TYPES" ]; then
+              echo "no GPU instance types advertised; nothing to reconcile"
+              exit 0
+            fi
+            IDS=$(AWS_PROFILE=spinifex aws ec2 describe-instances \
+              --filters "Name=instance-type,Values=${GPU_TYPES}" "Name=instance-state-name,Values=pending,running" \
+              --query "Reservations[].Instances[].InstanceId" --output text)
+            if [ -z "$IDS" ] || [ "$IDS" = "None" ]; then
+              echo "no leftover GPU instances — GPU already released"
+              exit 0
+            fi
+            echo "terminating leftover GPU instance(s): $IDS"
+            AWS_PROFILE=spinifex aws ec2 terminate-instances --instance-ids $IDS
+          '
+          ssh ${{ env.ssh_opts }} "$VM" "$REMOTE"
+
+      - name: Fail job if GPU suite failed
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
+            echo "::error::GPU e2e suite failed (rc=${{ steps.e2e.outputs.rc }}) — see analysis above; raw ES/APM traces at ${SPINIFEX_OTLP_ENDPOINT} (ci.run_id=${{ github.run_id }})"
+            exit 1
+          fi

.github/workflows/publish-eks-ami.yml

@@ -1,18 +1,34 @@
 name: Publish EKS + ECS node AMIs
 
-# Builds the eks-node and ecs-agent system AMIs (libguestfs, on the runner) and
-# publishes each compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket
-# that backs iso.mulgadc.com/system-ami/. Operators then install them on any
-# cluster with:
+# Builds the system AMIs (libguestfs, on the runner) and publishes each
+# compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
+# iso.mulgadc.com/system-ami/. Four images ship:
+#   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
+#   spinifex-ecs-node       Alpine ECS container instance
+#   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
+#   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
+# Operators then install them on any cluster with:
 #   spx admin images import --name spinifex-eks-node
 #   spx admin images import --name spinifex-ecs-node
+#   spx admin images import --name spinifex-eks-node-gpu
+#   spx admin images import --name spinifex-ecs-node-gpu
 #
 # Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
-# action, not something every push should trigger.
+# action, not something every push should trigger. The `variant` input scopes
+# which images build — the Ubuntu GPU legs are 16G and much slower (NVIDIA
+# driver download + DKMS build), so publishing only the CPU AMIs stays cheap.
 
 on:
   workflow_dispatch:
     inputs:
+      variant:
+        description: "Which images to build"
+        type: choice
+        default: all
+        options:
+          - all
+          - cpu
+          - gpu
       dry_run:
         description: "Build + convert only; skip the R2 upload"
         type: boolean
@@ -26,9 +42,37 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-  publish:
-    name: Build + publish eks-node + ecs-agent AMIs
+  matrix:
+    name: Resolve image matrix
     runs-on: [ self-hosted, ci-single ]
+    outputs:
+      images: ${{ steps.gen.outputs.images }}
+    steps:
+      - name: Select images for variant
+        id: gen
+        env:
+          VARIANT: ${{ inputs.variant }}
+        run: |
+          set -euo pipefail
+          cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
+          gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
+          case "${VARIANT}" in
+            cpu) items="${cpu}" ;;
+            gpu) items="${gpu}" ;;
+            *)   items="${cpu},${gpu}" ;;
+          esac
+          echo "images={\"include\":[${items}]}" >> "$GITHUB_OUTPUT"
+
+  publish:
+    name: Build + publish ${{ matrix.name }}
+    needs: matrix
+    runs-on: [ self-hosted, ci-single ]
+    strategy:
+      fail-fast: false
+      # ci-single is one runner and each GPU build needs ~16G of scratch disk;
+      # serialise the legs so they never contend for /_temp or the mirror.
+      max-parallel: 1
+      matrix: ${{ fromJSON(needs.matrix.outputs.images) }}
 
     steps:
       - name: Checkout Spinifex
@@ -75,7 +119,7 @@ jobs:
             rm -rf /tmp/awscliv2.zip /tmp/aws-install
           fi
 
-      - name: Build + publish eks-node to R2
+      - name: Build + publish ${{ matrix.name }} to R2
         working-directory: spinifex
         env:
           # Map the repo's R2 secrets onto the names publish-system-image.sh reads.
@@ -84,34 +128,20 @@ jobs:
           R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
           R2_BUCKET: spinifex-iso
           DRY_RUN: ${{ inputs.dry_run }}
+          MANIFEST: ${{ matrix.manifest }}
+          # Ubuntu GPU images are 16G — a RAM-backed /tmp cannot hold the qcow2 +
+          # expanded raw, so point the build scratch at the runner's on-disk
+          # _temp. Harmless for the small Alpine legs too.
+          SYSTEM_IMAGE_BUILD_DIR: ${{ runner.temp }}/system-image-build
         run: |
           set -euo pipefail
           # libguestfs needs to read a host kernel; Ubuntu ships it mode 0600.
           # direct backend = no libvirt on the runner.
           export LIBGUESTFS_BACKEND=direct
           sudo chmod 0644 /boot/vmlinuz-* || true
+          mkdir -p "${SYSTEM_IMAGE_BUILD_DIR}"
           FLAGS=(--build)
           if [[ "${DRY_RUN}" == "true" ]]; then
             FLAGS+=(--dry-run)
           fi
-          ./scripts/publish-system-image.sh \
-            scripts/images/eks-node/manifest.conf "${FLAGS[@]}"
-
-      - name: Build + publish ecs-agent to R2
-        working-directory: spinifex
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
-          R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
-          R2_BUCKET: spinifex-iso
-          DRY_RUN: ${{ inputs.dry_run }}
-        run: |
-          set -euo pipefail
-          export LIBGUESTFS_BACKEND=direct
-          sudo chmod 0644 /boot/vmlinuz-* || true
-          FLAGS=(--build)
-          if [[ "${DRY_RUN}" == "true" ]]; then
-            FLAGS+=(--dry-run)
-          fi
-          ./scripts/publish-system-image.sh \
-            scripts/images/ecs-agent/manifest.conf "${FLAGS[@]}"
+          ./scripts/publish-system-image.sh "${MANIFEST}" "${FLAGS[@]}"

README.md

@@ -192,6 +192,10 @@ While Spinifex is architected and implemented by experienced engineers, we lever
 
 This hybrid approach ensures Spinifex benefits from both proven engineering expertise and modern development acceleration, while maintaining the quality and reliability standards required for production infrastructure.
 
+## Trademarks
+
+"AWS", "Amazon Web Services", and all related service names (EC2, EBS, S3, VPC, IAM, and others) are trademarks of Amazon.com, Inc. or its affiliates. Mulga Defense Corporation and Spinifex are independent and not affiliated with, endorsed by, or sponsored by Amazon.com, Inc. or Amazon Web Services, Inc. References to AWS services describe interoperability and compatibility only.
+
 ## License
 
 Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.

cmd/ecs-agent/agent.go

@@ -43,6 +43,11 @@ type Agent struct {
 	// cred serves task IAM role credentials at 169.254.170.2 (nil in unit tests).
 	cred *credEndpoint
 
+	// gpu is the local pinning ledger over the host's discovered GPU UUIDs.
+	// Always non-nil (newAgent seeds an empty one); New() reseeds it from
+	// nvidia-smi discovery.
+	gpu *gpuLedger
+
 	closers []func() error
 }
 
@@ -61,6 +66,7 @@ func newAgent(cfg config, id identity, cp controlPlane, puller ctrruntime.ImageP
 		reg:      newRegistrar(cp, id),
 		hb:       newHeartbeater(cp, id, cfg.Heartbeat),
 		netns:    newTaskNetns(execNetRunner{}),
+		gpu:      newGPULedger(id.Capacity.GPUIDs),
 	}
 }
 
@@ -84,7 +90,7 @@ func New(cfg config) (*Agent, error) {
 		InstanceID:   meta.InstanceID,
 		AZ:           meta.AZ,
 		Hostname:     host,
-		Capacity:     detectCapacity(),
+		Capacity:     detectCapacity(discoverNvidiaGPUs(execCommandRunner)),
 		AgentVersion: version,
 	}
 
@@ -153,11 +159,15 @@ func (a *Agent) Stop() error {
 	return firstErr
 }
 
-// detectCapacity reports the host's total CPU units (1 vCPU = 1024) and memory.
-func detectCapacity() bus.InstanceCapacity {
+// detectCapacity reports the host's total CPU units (1 vCPU = 1024), memory,
+// and GPU count/UUIDs from the caller's nvidia-smi discovery (nil on a
+// non-GPU host).
+func detectCapacity(gpuUUIDs []string) bus.InstanceCapacity {
 	return bus.InstanceCapacity{
 		CPU:       goruntime.NumCPU() * 1024,
 		MemoryMiB: memTotalMiB(),
+		GPU:       len(gpuUUIDs),
+		GPUIDs:    gpuUUIDs,
 	}
 }
 

cmd/ecs-agent/agent_test.go

@@ -42,7 +42,7 @@ func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
 }
 
 func TestDetectCapacity_Positive(t *testing.T) {
-	c := detectCapacity()
+	c := detectCapacity(nil)
 	if c.CPU <= 0 {
 		t.Errorf("CPU = %d, want > 0", c.CPU)
 	}
@@ -50,4 +50,17 @@ func TestDetectCapacity_Positive(t *testing.T) {
 	if c.MemoryMiB < 0 {
 		t.Errorf("MemoryMiB = %d, want >= 0", c.MemoryMiB)
 	}
+	if c.GPU != 0 || c.GPUIDs != nil {
+		t.Errorf("no GPU UUIDs given: want GPU=0/GPUIDs=nil, got %d/%v", c.GPU, c.GPUIDs)
+	}
+}
+
+func TestDetectCapacity_WithGPUUUIDs(t *testing.T) {
+	c := detectCapacity([]string{"GPU-aaa", "GPU-bbb"})
+	if c.GPU != 2 {
+		t.Errorf("GPU = %d, want 2", c.GPU)
+	}
+	if len(c.GPUIDs) != 2 {
+		t.Errorf("GPUIDs = %v, want 2 entries", c.GPUIDs)
+	}
 }

cmd/ecs-agent/controlplane.go

@@ -29,6 +29,9 @@ type controlPlane interface {
 	// assign taskIDs and stop taskIDs accepted on the previous poll. Returns the
 	// still-pending assignments and stop directives.
 	PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error)
+	// ReportTaskGPU reports the local ledger's pinned device UUIDs for a task's
+	// containers, so the control plane can populate DescribeTasks gpuIds.
+	ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error
 }
 
 // gatewayControlPlane implements controlPlane over the SigV4 ECS gateway client.
@@ -74,6 +77,11 @@ func (g *gatewayControlPlane) Register(id identity) error {
 		},
 		VersionInfo: &ecs.VersionInfo{AgentVersion: aws.String(id.AgentVersion)},
 	}
+	if len(id.Capacity.GPUIDs) > 0 {
+		in.TotalResources = append(in.TotalResources, &ecs.Resource{
+			Name: aws.String("GPU"), Type: aws.String("STRINGSET"), StringSetValue: aws.StringSlice(id.Capacity.GPUIDs),
+		})
+	}
 	body, err := json.Marshal(in)
 	if err != nil {
 		return fmt.Errorf("marshal register: %w", err)
@@ -133,3 +141,15 @@ func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssig
 	}
 	return out.Assignments, out.Stops, nil
 }
+
+func (g *gatewayControlPlane) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
+	in := &handlers_ecs.ReportTaskGPUInput{Cluster: cluster, Task: task, Containers: containers}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("marshal report-task-gpu: %w", err)
+	}
+	if _, err := g.client.Call("ReportTaskGPU", body); err != nil {
+		return fmt.Errorf("report task gpu: %w", err)
+	}
+	return nil
+}

cmd/ecs-agent/controlplane_test.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
+)
+
+// fixedCredsProvider is a static credentials.CredentialsProvider for tests
+// that only need a valid SigV4 signature, not real IMDS rotation.
+type fixedCredsProvider struct{}
+
+func (fixedCredsProvider) Retrieve(context.Context) (credentials.Credentials, error) {
+	return credentials.Credentials{AccessKeyID: "AKIATEST", SecretAccessKey: "secret"}, nil
+}
+
+func newTestGatewayControlPlane(t *testing.T, srv *httptest.Server) *gatewayControlPlane {
+	t.Helper()
+	cp, err := newGatewayControlPlane(config{GatewayURL: srv.URL, Region: "us-east-1"}, fixedCredsProvider{})
+	if err != nil {
+		t.Fatalf("newGatewayControlPlane: %v", err)
+	}
+	return cp
+}
+
+// Register carries the discovered GPU UUIDs as a STRINGSET resource, matching
+// the control plane's RegisterContainerInstance GPU parsing (Epic C3).
+func TestGatewayControlPlane_Register_SendsGPUStringSet(t *testing.T) {
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	id := identity{
+		ClusterName: "default", InstanceID: "i-1",
+		Capacity: detectCapacity([]string{"GPU-aaa", "GPU-bbb"}),
+	}
+	if err := cp.Register(id); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+
+	var in ecs.RegisterContainerInstanceInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	var gpu *ecs.Resource
+	for _, r := range in.TotalResources {
+		if r.Name != nil && *r.Name == "GPU" {
+			gpu = r
+		}
+	}
+	if gpu == nil {
+		t.Fatal("no GPU resource in TotalResources")
+	}
+	if *gpu.Type != "STRINGSET" {
+		t.Errorf("GPU resource type = %q, want STRINGSET", *gpu.Type)
+	}
+	got := make([]string, len(gpu.StringSetValue))
+	for i, v := range gpu.StringSetValue {
+		got[i] = *v
+	}
+	if len(got) != 2 || got[0] != "GPU-aaa" || got[1] != "GPU-bbb" {
+		t.Errorf("GPU UUIDs = %v, want [GPU-aaa GPU-bbb]", got)
+	}
+}
+
+// A non-GPU host (no discovered UUIDs) registers with no GPU resource at all.
+func TestGatewayControlPlane_Register_NoGPUOmitsResource(t *testing.T) {
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	id := identity{ClusterName: "default", InstanceID: "i-1", Capacity: detectCapacity(nil)}
+	if err := cp.Register(id); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+
+	var in ecs.RegisterContainerInstanceInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	for _, r := range in.TotalResources {
+		if r.Name != nil && *r.Name == "GPU" {
+			t.Fatalf("unexpected GPU resource on a non-GPU host: %+v", r)
+		}
+	}
+}
+
+// ReportTaskGPU posts the per-container device UUIDs under the ReportTaskGPU
+// action, decodable as the handlers_ecs internal shape.
+func TestGatewayControlPlane_ReportTaskGPU(t *testing.T) {
+	var gotTarget string
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotTarget = r.Header.Get("X-Amz-Target")
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	err := cp.ReportTaskGPU("web", "t-1", []handlers_ecs.ContainerGPUReport{
+		{Name: "trainer", GPUIDs: []string{"GPU-aaa"}},
+	})
+	if err != nil {
+		t.Fatalf("ReportTaskGPU: %v", err)
+	}
+	if gotTarget == "" {
+		t.Fatal("no X-Amz-Target header sent")
+	}
+
+	var in handlers_ecs.ReportTaskGPUInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	if in.Cluster != "web" || in.Task != "t-1" {
+		t.Errorf("cluster/task = %q/%q, want web/t-1", in.Cluster, in.Task)
+	}
+	if len(in.Containers) != 1 || in.Containers[0].Name != "trainer" || len(in.Containers[0].GPUIDs) != 1 {
+		t.Errorf("containers = %+v, want one trainer entry with one UUID", in.Containers)
+	}
+}

cmd/ecs-agent/gpu_discovery.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"os/exec"
+	"strings"
+)
+
+// commandRunner abstracts exec.Command so discovery is unit-testable without
+// shelling out. Production uses execCommandRunner; tests inject a fake.
+type commandRunner func(name string, args ...string) ([]byte, error)
+
+// execCommandRunner runs a real host command and returns its stdout.
+func execCommandRunner(name string, args ...string) ([]byte, error) {
+	return exec.Command(name, args...).Output()
+}
+
+// discoverNvidiaGPUs enumerates local NVIDIA GPU device UUIDs via nvidia-smi.
+// A missing binary or any run failure (no GPU host) yields an empty list, not
+// an error — the agent still registers with zero GPU capacity.
+func discoverNvidiaGPUs(run commandRunner) []string {
+	out, err := run("nvidia-smi", "--query-gpu=uuid", "--format=csv,noheader")
+	if err != nil {
+		return nil
+	}
+	return parseNvidiaSMIUUIDs(string(out))
+}
+
+// parseNvidiaSMIUUIDs splits nvidia-smi's newline-delimited UUID output,
+// dropping blank lines. Exported as a pure function so fixture strings can be
+// tested without exec.
+func parseNvidiaSMIUUIDs(out string) []string {
+	lines := strings.Split(strings.TrimSpace(out), "\n")
+	uuids := make([]string, 0, len(lines))
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		uuids = append(uuids, line)
+	}
+	return uuids
+}

cmd/ecs-agent/gpu_discovery_test.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+)
+
+func TestParseNvidiaSMIUUIDs(t *testing.T) {
+	cases := []struct {
+		name string
+		in   string
+		want []string
+	}{
+		{"empty", "", nil},
+		{"whitespace only", "   \n\n  ", nil},
+		{"single", "GPU-11111111-1111-1111-1111-111111111111\n", []string{"GPU-11111111-1111-1111-1111-111111111111"}},
+		{"multi", "GPU-aaa\nGPU-bbb\nGPU-ccc\n", []string{"GPU-aaa", "GPU-bbb", "GPU-ccc"}},
+		{"blank lines between", "GPU-aaa\n\nGPU-bbb\n", []string{"GPU-aaa", "GPU-bbb"}},
+		{"trailing whitespace per line", "GPU-aaa \r\nGPU-bbb\r\n", []string{"GPU-aaa", "GPU-bbb"}},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got := parseNvidiaSMIUUIDs(c.in)
+			if !reflect.DeepEqual(got, c.want) && (len(got) != 0 || len(c.want) != 0) {
+				t.Errorf("parseNvidiaSMIUUIDs(%q) = %v, want %v", c.in, got, c.want)
+			}
+		})
+	}
+}
+
+func TestDiscoverNvidiaGPUs_AbsentBinaryReturnsEmptyNoError(t *testing.T) {
+	run := func(name string, args ...string) ([]byte, error) {
+		return nil, errors.New(`exec: "nvidia-smi": executable file not found in $PATH`)
+	}
+	got := discoverNvidiaGPUs(run)
+	if len(got) != 0 {
+		t.Errorf("want empty list on absent nvidia-smi, got %v", got)
+	}
+}
+
+func TestDiscoverNvidiaGPUs_ParsesFixtureOutput(t *testing.T) {
+	var gotName string
+	var gotArgs []string
+	run := func(name string, args ...string) ([]byte, error) {
+		gotName = name
+		gotArgs = args
+		return []byte("GPU-aaa\nGPU-bbb\n"), nil
+	}
+	got := discoverNvidiaGPUs(run)
+	want := []string{"GPU-aaa", "GPU-bbb"}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+	if gotName != "nvidia-smi" {
+		t.Errorf("command = %q, want nvidia-smi", gotName)
+	}
+	if len(gotArgs) != 2 || gotArgs[0] != "--query-gpu=uuid" || gotArgs[1] != "--format=csv,noheader" {
+		t.Errorf("args = %v, want the uuid/csv,noheader query", gotArgs)
+	}
+}
+
+func TestDiscoverNvidiaGPUs_NoDevicesReturnsEmpty(t *testing.T) {
+	run := func(name string, args ...string) ([]byte, error) {
+		return []byte(""), nil
+	}
+	got := discoverNvidiaGPUs(run)
+	if len(got) != 0 {
+		t.Errorf("want empty list, got %v", got)
+	}
+}

cmd/ecs-agent/gpu_ledger.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+)
+
+// gpuLedger is the agent's local source of truth for which of the host's
+// discovered GPU UUIDs are free vs pinned to a running task's container. C4
+// reads a task's pinned UUIDs to build the CDI device list for containerd.
+type gpuLedger struct {
+	mu     sync.Mutex
+	free   []string
+	pinned map[string][]string // key ("taskID/containerName") -> uuids
+}
+
+// newGPULedger seeds the ledger with the host's discovered device UUIDs. A nil
+// or empty slice is a valid non-GPU host: every Pin then fails, which callers
+// treat as best-effort (no device, no CDI injection later).
+func newGPULedger(uuids []string) *gpuLedger {
+	free := append([]string(nil), uuids...)
+	return &gpuLedger{free: free, pinned: make(map[string][]string)}
+}
+
+// gpuKey builds the ledger key for one container's pin/release.
+func gpuKey(taskID, containerName string) string {
+	return taskID + "/" + containerName
+}
+
+// Pin reserves n free UUIDs under key, returning them. An error means fewer
+// than n devices were free; the key is left unpinned so a retry (or a
+// best-effort caller) can proceed without a partial reservation.
+func (l *gpuLedger) Pin(key string, n int) ([]string, error) {
+	if n <= 0 {
+		return nil, nil
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if len(l.free) < n {
+		return nil, fmt.Errorf("gpu ledger: requested %d GPUs, %d free", n, len(l.free))
+	}
+	uuids := append([]string(nil), l.free[:n]...)
+	l.free = l.free[n:]
+	l.pinned[key] = uuids
+	return uuids, nil
+}
+
+// ReleaseTask frees every UUID pinned under any container key belonging to
+// taskID, called once the task's terminal STOPPED state is reported.
+func (l *gpuLedger) ReleaseTask(taskID string) {
+	prefix := taskID + "/"
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	for key, uuids := range l.pinned {
+		if !strings.HasPrefix(key, prefix) {
+			continue
+		}
+		l.free = append(l.free, uuids...)
+		delete(l.pinned, key)
+	}
+}

cmd/ecs-agent/gpu_ledger_test.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"sync"
+	"testing"
+)
+
+func TestGPULedger_PinReturnsDistinctUUIDs(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b", "GPU-c"})
+	got, err := l.Pin("t-1/web", 2)
+	if err != nil {
+		t.Fatalf("Pin: %v", err)
+	}
+	if len(got) != 2 {
+		t.Fatalf("want 2 UUIDs, got %v", got)
+	}
+	if got[0] == got[1] {
+		t.Errorf("pinned UUIDs must be distinct, got %v", got)
+	}
+	if len(l.free) != 1 {
+		t.Errorf("free = %d, want 1 remaining", len(l.free))
+	}
+}
+
+func TestGPULedger_PinInsufficientFreeErrors(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a"})
+	_, err := l.Pin("t-1/web", 2)
+	if err == nil {
+		t.Fatal("want error when requesting more GPUs than free")
+	}
+	if len(l.free) != 1 {
+		t.Errorf("failed pin must not consume any device, free = %d", len(l.free))
+	}
+}
+
+func TestGPULedger_PinZeroIsNoop(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a"})
+	got, err := l.Pin("t-1/web", 0)
+	if err != nil || got != nil {
+		t.Errorf("Pin(0) = %v, %v; want nil, nil", got, err)
+	}
+}
+
+func TestGPULedger_ReleaseTaskFreesOnlyThatTasksDevices(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b", "GPU-c", "GPU-d"})
+	if _, err := l.Pin(gpuKey("t-1", "web"), 2); err != nil {
+		t.Fatalf("pin t-1: %v", err)
+	}
+	if _, err := l.Pin(gpuKey("t-2", "worker"), 1); err != nil {
+		t.Fatalf("pin t-2: %v", err)
+	}
+	if len(l.free) != 1 {
+		t.Fatalf("free = %d, want 1 before release", len(l.free))
+	}
+
+	l.ReleaseTask("t-1")
+	if len(l.free) != 3 {
+		t.Errorf("free = %d, want 3 after releasing t-1's two devices", len(l.free))
+	}
+	if _, ok := l.pinned[gpuKey("t-2", "worker")]; !ok {
+		t.Error("t-2's pin should survive releasing t-1")
+	}
+	if _, ok := l.pinned[gpuKey("t-1", "web")]; ok {
+		t.Error("t-1's pin should be gone after release")
+	}
+}
+
+// Multi-device pinning across concurrent task assigns must hand out disjoint
+// UUID sets — no two tasks may ever share a device.
+func TestGPULedger_ConcurrentPinsHandOutDistinctUUIDs(t *testing.T) {
+	const nDevices = 20
+	uuids := make([]string, nDevices)
+	for i := range uuids {
+		uuids[i] = "GPU-" + string(rune('a'+i))
+	}
+	l := newGPULedger(uuids)
+
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	seen := make(map[string]bool)
+	errs := 0
+	for i := range nDevices {
+		wg.Add(1)
+		go func(n int) {
+			defer wg.Done()
+			key := gpuKey("t-concurrent", string(rune('A'+n)))
+			got, err := l.Pin(key, 1)
+			mu.Lock()
+			defer mu.Unlock()
+			if err != nil {
+				errs++
+				return
+			}
+			for _, u := range got {
+				if seen[u] {
+					t.Errorf("UUID %s handed out to more than one concurrent pin", u)
+				}
+				seen[u] = true
+			}
+		}(i)
+	}
+	wg.Wait()
+	if errs != 0 {
+		t.Errorf("want all %d pins to succeed (exactly enough devices), got %d errors", nDevices, errs)
+	}
+	if len(seen) != nDevices {
+		t.Errorf("distinct UUIDs handed out = %d, want %d", len(seen), nDevices)
+	}
+	if len(l.free) != 0 {
+		t.Errorf("free = %d, want 0 after all devices pinned", len(l.free))
+	}
+}
+
+// Releasing a task's devices concurrently with new pins never double-hands a
+// UUID: a device is only ever free or pinned to exactly one key at a time.
+func TestGPULedger_ConcurrentPinAndReleaseNoDoubleAssignment(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b"})
+	var wg sync.WaitGroup
+	for round := range 50 {
+		wg.Add(2)
+		taskID := "t-" + string(rune('a'+round%26))
+		go func(id string) {
+			defer wg.Done()
+			_, _ = l.Pin(gpuKey(id, "web"), 1)
+		}(taskID)
+		go func(id string) {
+			defer wg.Done()
+			l.ReleaseTask(id)
+		}(taskID)
+	}
+	wg.Wait()
+	// Invariant: every currently-pinned UUID appears in exactly one pinned entry
+	// and never simultaneously in free.
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	freeSet := make(map[string]bool, len(l.free))
+	for _, u := range l.free {
+		if freeSet[u] {
+			t.Errorf("duplicate UUID %s in free list", u)
+		}
+		freeSet[u] = true
+	}
+	pinnedSet := make(map[string]bool)
+	for _, uuids := range l.pinned {
+		for _, u := range uuids {
+			if pinnedSet[u] {
+				t.Errorf("UUID %s pinned under more than one key", u)
+			}
+			if freeSet[u] {
+				t.Errorf("UUID %s is both free and pinned", u)
+			}
+			pinnedSet[u] = true
+		}
+	}
+}

cmd/ecs-agent/registration_test.go

@@ -7,16 +7,19 @@ import (
 	"testing"
 	"time"
 
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )
 
-// fakeCP is a test double for the gateway control-plane. It records registers and
-// task-state reports, and replays a scripted queue of poll responses.
+// fakeCP is a test double for the gateway control-plane. It records registers,
+// task-state reports and GPU reports, and replays a scripted queue of poll
+// responses.
 type fakeCP struct {
 	mu sync.Mutex
 
 	registers    int
 	states       []bus.TaskState
+	gpuReports   []gpuReport
 	pollAcks     [][]string
 	pollStopAcks [][]string
 	pollReplies  [][]bus.Assign
@@ -27,6 +30,13 @@ type fakeCP struct {
 	submitErr   bool
 }
 
+// gpuReport is one recorded ReportTaskGPU call.
+type gpuReport struct {
+	cluster    string
+	task       string
+	containers []handlers_ecs.ContainerGPUReport
+}
+
 var _ controlPlane = (*fakeCP)(nil)
 
 func (f *fakeCP) Register(_ identity) error {
@@ -66,6 +76,25 @@ func (f *fakeCP) PollAssignments(_, _ string, ackAssigns, ackStops []string) ([]
 	return out, stops, nil
 }
 
+func (f *fakeCP) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.gpuReports = append(f.gpuReports, gpuReport{cluster: cluster, task: task, containers: containers})
+	return nil
+}
+
+func (f *fakeCP) gpuReportsFor(taskID string) []gpuReport {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []gpuReport
+	for _, r := range f.gpuReports {
+		if r.task == taskID {
+			out = append(out, r)
+		}
+	}
+	return out
+}
+
 func (f *fakeCP) registerCount() int {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -102,6 +131,15 @@ func testIdentity() identity {
 	}
 }
 
+// testIdentityWithGPUs is testIdentity seeded with discovered GPU UUIDs, so
+// newAgent's ledger has real devices to pin from.
+func testIdentityWithGPUs(uuids ...string) identity {
+	id := testIdentity()
+	id.Capacity.GPU = len(uuids)
+	id.Capacity.GPUIDs = uuids
+	return id
+}
+
 func TestRegistrar_Register(t *testing.T) {
 	cp := &fakeCP{}
 	if err := newRegistrar(cp, testIdentity()).Register(); err != nil {

cmd/ecs-agent/runtime/containerd_run.go

@@ -6,12 +6,41 @@ import (
 	"sort"
 
 	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/pkg/cdi"
 	"github.com/containerd/containerd/v2/pkg/cio"
 	"github.com/containerd/containerd/v2/pkg/namespaces"
 	"github.com/containerd/containerd/v2/pkg/oci"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
+// cdiVendorGPU is the NVIDIA CDI vendor/class prefix for whole-GPU devices, per
+// the NVIDIA CDI naming convention (nvidia.com/gpu=<uuid>).
+const cdiVendorGPU = "nvidia.com/gpu"
+
+// cdiDeviceNames maps a container's pinned GPU device UUIDs to CDI device
+// names. Empty input (non-GPU container) returns nil so no CDI opt is added.
+func cdiDeviceNames(uuids []string) []string {
+	if len(uuids) == 0 {
+		return nil
+	}
+	names := make([]string, len(uuids))
+	for i, id := range uuids {
+		names[i] = cdiVendorGPU + "=" + id
+	}
+	return names
+}
+
+// cdiSpecOpts returns the oci.SpecOpts needed to inject gpuIDs (via
+// containerd's CDI machinery) as devices into a container's OCI spec. It
+// returns nil for a non-GPU container so Run adds nothing to the opts list.
+func cdiSpecOpts(gpuIDs []string) []oci.SpecOpts {
+	devices := cdiDeviceNames(gpuIDs)
+	if len(devices) == 0 {
+		return nil
+	}
+	return []oci.SpecOpts{cdi.WithCDIDevices(devices...)}
+}
+
 var _ Runner = (*containerdPuller)(nil)
 
 // Run creates a container from an already-pulled image and starts its task. The
@@ -51,6 +80,7 @@ func (p *containerdPuller) Run(ctx context.Context, id string, spec RunSpec) (st
 	if len(spec.Env) > 0 {
 		specOpts = append(specOpts, oci.WithEnv(envSlice(spec.Env)))
 	}
+	specOpts = append(specOpts, cdiSpecOpts(spec.GPUIDs)...)
 
 	container, err := p.client.NewContainer(ctx, id,
 		containerd.WithNewSnapshot(id+"-snapshot", image),

cmd/ecs-agent/runtime/containerd_run_test.go

@@ -0,0 +1,47 @@
+package runtime
+
+import (
+	"reflect"
+	"testing"
+)
+
+// cdiDeviceNames maps a container's pinned GPU UUIDs to CDI device names
+// (nvidia.com/gpu=<uuid>); Run wires the result into oci.WithCDIDevices when
+// non-empty, so this is the injection logic's unit-testable core.
+func TestCDIDeviceNames(t *testing.T) {
+	cases := []struct {
+		name  string
+		uuids []string
+		want  []string
+	}{
+		{name: "empty", uuids: nil, want: nil},
+		{name: "single", uuids: []string{"GPU-aaaaaaaa-1111-2222-3333-444444444444"},
+			want: []string{"nvidia.com/gpu=GPU-aaaaaaaa-1111-2222-3333-444444444444"}},
+		{name: "multiple preserves order", uuids: []string{"GPU-aaa", "GPU-bbb"},
+			want: []string{"nvidia.com/gpu=GPU-aaa", "nvidia.com/gpu=GPU-bbb"}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := cdiDeviceNames(tc.uuids)
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("cdiDeviceNames(%v) = %v, want %v", tc.uuids, got, tc.want)
+			}
+		})
+	}
+}
+
+// TestCDISpecOpts_Wiring confirms Run's specOpts list picks up exactly one CDI
+// opt when GPUIDs is non-empty, and none for a non-GPU container — the wiring
+// Run relies on (a live oci.Spec is needed to unit test the opt's effect).
+func TestCDISpecOpts_Wiring(t *testing.T) {
+	if opts := cdiSpecOpts(nil); len(opts) != 0 {
+		t.Errorf("non-GPU container: want no spec opts, got %d", len(opts))
+	}
+	opts := cdiSpecOpts([]string{"GPU-aaa", "GPU-bbb"})
+	if len(opts) != 1 {
+		t.Fatalf("GPU container: want exactly one CDI spec opt, got %d", len(opts))
+	}
+	if opts[0] == nil {
+		t.Error("GPU container: spec opt is nil")
+	}
+}

cmd/ecs-agent/runtime/runtime.go

@@ -48,6 +48,12 @@ type RunSpec struct {
 	// (awsvpc task ENI). Empty keeps the container in the host (VM) netns —
 	// bridge/host mode behaviour.
 	NetnsPath string
+	// GPU is the whole-GPU count requested for this container.
+	GPU int
+	// GPUIDs are the ledger-pinned device UUIDs assigned to this container, one
+	// per GPU requested. Empty for non-GPU containers or if pinning fell short.
+	// The runner injects these as CDI devices into the OCI spec.
+	GPUIDs []string
 }
 
 // RunStatus is a finished container's outcome.

cmd/ecs-agent/taskrun.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )
 
@@ -124,12 +125,15 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 		}
 
 		cid := containerID(as.TaskID, c.Name)
+		gpuIDs := a.pinContainerGPUs(as.TaskID, c.Name, c.GPU)
 		spec := ctrruntime.RunSpec{
 			Image:     c.Image,
 			Command:   c.Command,
 			Env:       withCredEnv(c.Environment, credID),
 			Labels:    taskLabels(as, c.Name),
 			NetnsPath: netnsPath,
+			GPU:       c.GPU,
+			GPUIDs:    gpuIDs,
 		}
 		id, err := a.runner.Run(ctx, cid, spec)
 		if err != nil {
@@ -138,7 +142,9 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 			a.reportTaskState(as, bus.TaskStatusStopped, "container start failed: "+err.Error(), statuses)
 			return
 		}
-		statuses = append(statuses, bus.ContainerStatus{Name: c.Name, Status: bus.TaskStatusRunning, ContainerID: id})
+		statuses = append(statuses, bus.ContainerStatus{
+			Name: c.Name, Status: bus.TaskStatusRunning, ContainerID: id, GPUIDs: gpuIDs,
+		})
 		go a.waitContainer(ctx, as, c.Name, id)
 	}
 
@@ -234,8 +240,26 @@ func withCredEnv(env map[string]string, credID string) map[string]string {
 	return out
 }
 
+// pinContainerGPUs reserves n device UUIDs for a container from the local
+// ledger, before the container is started so the runner can inject them as
+// CDI devices. A short ledger (fewer free devices than requested, e.g.
+// discovery found none) is logged and treated as best-effort: the container
+// still runs without pinned UUIDs (and without GPU access) rather than
+// failing the task.
+func (a *Agent) pinContainerGPUs(taskID, containerName string, n int) []string {
+	if n <= 0 || a.gpu == nil {
+		return nil
+	}
+	uuids, err := a.gpu.Pin(gpuKey(taskID, containerName), n)
+	if err != nil {
+		slog.Warn("ecs-agent: gpu pin short", "task", taskID, "container", containerName, "err", err)
+	}
+	return uuids
+}
+
 // reportTaskState reports a task transition through the gateway's
-// SubmitTaskStateChange action.
+// SubmitTaskStateChange action, then (RUNNING with pinned GPUs) reports the
+// per-container device UUIDs, and (STOPPED) releases them back to the ledger.
 func (a *Agent) reportTaskState(as *bus.Assign, status, reason string, containers []bus.ContainerStatus) {
 	if a.cp == nil {
 		return
@@ -253,6 +277,28 @@ func (a *Agent) reportTaskState(as *bus.Assign, status, reason string, container
 	if err := a.cp.SubmitTaskState(msg); err != nil {
 		slog.Error("ecs-agent: report task-state failed", "task", as.TaskID, "err", err)
 	}
+	a.reportTaskGPUs(as.TaskID, containers)
+	if status == bus.TaskStatusStopped && a.gpu != nil {
+		a.gpu.ReleaseTask(as.TaskID)
+	}
+}
+
+// reportTaskGPUs sends the control plane the pinned device UUIDs for any
+// container in the report that has them, so DescribeTasks can surface gpuIds.
+// A no-op when nothing was pinned (the common non-GPU-task case).
+func (a *Agent) reportTaskGPUs(taskID string, containers []bus.ContainerStatus) {
+	reports := make([]handlers_ecs.ContainerGPUReport, 0, len(containers))
+	for _, c := range containers {
+		if len(c.GPUIDs) > 0 {
+			reports = append(reports, handlers_ecs.ContainerGPUReport{Name: c.Name, GPUIDs: c.GPUIDs})
+		}
+	}
+	if len(reports) == 0 {
+		return
+	}
+	if err := a.cp.ReportTaskGPU(a.id.ClusterName, taskID, reports); err != nil {
+		slog.Error("ecs-agent: report task-gpu failed", "task", taskID, "err", err)
+	}
 }
 
 // containerID composes a containerd-valid container ID from a task + container.

cmd/ecs-agent/taskrun_test.go

@@ -70,6 +70,160 @@ func TestRunTask_PullRunReportsRunning(t *testing.T) {
 	}
 }
 
+// TestRunTask_GPUCarriedToRunSpec covers mulga-11opz (Epic C task C1): the
+// AssignContainer's GPU count reaches the runtime RunSpec unchanged; a
+// container with no GPU request keeps RunSpec.GPU at zero (regression).
+func TestRunTask_GPUCarriedToRunSpec(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 2,
+	})
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 2 {
+		t.Fatalf("expected two runs, got %+v", rt.Runs)
+	}
+	if rt.Runs[0].GPU != 0 {
+		t.Errorf("web container: want GPU=0, got %d", rt.Runs[0].GPU)
+	}
+	if rt.Runs[1].GPU != 2 {
+		t.Errorf("trainer container: want GPU=2, got %d", rt.Runs[1].GPU)
+	}
+}
+
+// TestRunTask_GPUPinsAndReportsUUIDs covers C3: a container's GPU request is
+// pinned from the agent's local ledger, carried on the RUNNING report's
+// ContainerStatus, and forwarded to the control plane via ReportTaskGPU.
+func TestRunTask_GPUPinsAndReportsUUIDs(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
+	})
+	a.runTask(context.Background(), as)
+
+	st := cp.taskStates()
+	if len(st) == 0 || st[len(st)-1].LastStatus != bus.TaskStatusRunning {
+		t.Fatalf("want RUNNING final state, got %+v", st)
+	}
+	last := st[len(st)-1]
+	var trainer *bus.ContainerStatus
+	for i := range last.Containers {
+		if last.Containers[i].Name == "trainer" {
+			trainer = &last.Containers[i]
+		}
+	}
+	if trainer == nil || len(trainer.GPUIDs) != 1 {
+		t.Fatalf("trainer container missing pinned GPU UUID: %+v", last.Containers)
+	}
+	if trainer.GPUIDs[0] != "GPU-aaa" && trainer.GPUIDs[0] != "GPU-bbb" {
+		t.Errorf("unexpected pinned UUID %q", trainer.GPUIDs[0])
+	}
+
+	reports := cp.gpuReportsFor(as.TaskID)
+	if len(reports) != 1 {
+		t.Fatalf("want one ReportTaskGPU call, got %d", len(reports))
+	}
+	if reports[0].cluster != as.ClusterName {
+		t.Errorf("report cluster = %q, want %q", reports[0].cluster, as.ClusterName)
+	}
+	if len(reports[0].containers) != 1 || reports[0].containers[0].Name != "trainer" {
+		t.Errorf("report containers = %+v, want one trainer entry", reports[0].containers)
+	}
+}
+
+// TestRunTask_GPUIDsReachRunSpec covers C4: the ledger's pinned UUIDs for a
+// container reach RunSpec.GPUIDs (not just the RunSpec.GPU count), so the
+// runner can inject them as CDI devices; a non-GPU container's RunSpec
+// carries no GPUIDs.
+func TestRunTask_GPUIDsReachRunSpec(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
+	})
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 2 {
+		t.Fatalf("expected two runs, got %+v", rt.Runs)
+	}
+	if len(rt.Runs[0].GPUIDs) != 0 {
+		t.Errorf("web container: want no GPUIDs, got %+v", rt.Runs[0].GPUIDs)
+	}
+	trainer := rt.Runs[1]
+	if len(trainer.GPUIDs) != 1 {
+		t.Fatalf("trainer container: want one pinned GPUID, got %+v", trainer.GPUIDs)
+	}
+	if trainer.GPUIDs[0] != "GPU-aaa" && trainer.GPUIDs[0] != "GPU-bbb" {
+		t.Errorf("unexpected pinned UUID %q", trainer.GPUIDs[0])
+	}
+}
+
+// A container with no GPU request never triggers a ReportTaskGPU call.
+func TestRunTask_NoGPUSkipsReport(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	a.runTask(context.Background(), as)
+
+	if reports := cp.gpuReportsFor(as.TaskID); len(reports) != 0 {
+		t.Errorf("want no GPU report for a non-GPU task, got %+v", reports)
+	}
+}
+
+// A GPU request exceeding the ledger's free devices (discovery found none, or
+// fewer than requested) is best-effort: the container still runs, it just
+// carries no pinned UUIDs and so no CDI devices are injected.
+func TestRunTask_GPUShortLedgerStillRuns(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt) // testIdentity() has no discovered GPU UUIDs
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 2,
+	})
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 2 {
+		t.Fatalf("expected both containers to run despite the short ledger, got %+v", rt.Runs)
+	}
+	if reports := cp.gpuReportsFor(as.TaskID); len(reports) != 0 {
+		t.Errorf("want no GPU report when nothing was pinned, got %+v", reports)
+	}
+}
+
+// stopTask releases the task's pinned GPU UUIDs back to the ledger so a later
+// task on the same instance can reuse them.
+func TestStopTask_ReleasesGPUsBackToLedger(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-trainer", Running: true, Labels: map[string]string{
+			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "trainer",
+		}},
+	}}
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa"), cp, rt, rt, nil)
+	if _, err := a.gpu.Pin(gpuKey("t-001", "trainer"), 1); err != nil {
+		t.Fatalf("seed pin: %v", err)
+	}
+	if len(a.gpu.free) != 0 {
+		t.Fatalf("setup: want ledger fully pinned, free = %v", a.gpu.free)
+	}
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
+
+	if len(a.gpu.free) != 1 {
+		t.Errorf("want the GPU released back to the ledger, free = %v", a.gpu.free)
+	}
+}
+
 // A pull failure reports STOPPED and never starts the container.
 func TestRunTask_PullFailureReportsStopped(t *testing.T) {
 	cp := &fakeCP{}

cmd/eks-gateway-fetch/main.go

@@ -156,9 +156,10 @@ func emitAddonsTSV(out io.Writer, body []byte) error {
 // recoveryDirective mirrors the gateway's RecoveryDirective wire shape; duplicated
 // here to keep this tiny VM binary free of the handler package's dependency graph.
 type recoveryDirective struct {
-	Epoch    int64  `json:"epoch"`
-	Action   string `json:"action"`
-	Snapshot string `json:"snapshot"`
+	Epoch            int64  `json:"epoch"`
+	Action           string `json:"action"`
+	Snapshot         string `json:"snapshot"`
+	SnapshotRequired bool   `json:"snapshotRequired"`
 }
 
 type internalRecoveryResponse struct {
@@ -166,8 +167,10 @@ type internalRecoveryResponse struct {
 }
 
 // emitRecoveryTSV decodes the recovery directive and writes a single
-// epoch\taction\tsnapshot line the on-VM k3s-recovery agent consumes without a
-// JSON parser. An unset action defaults to "none" (steady state).
+// epoch\taction\tsnapshot\tsnapshotRequired line the on-VM k3s-recovery agent
+// consumes without a JSON parser. An unset action defaults to "none" (steady
+// state); snapshotRequired is 1 when the snapshot MUST restore (fresh DR seed
+// aborts boot on fetch failure) else 0.
 func emitRecoveryTSV(out io.Writer, body []byte) error {
 	var resp internalRecoveryResponse
 	if err := json.Unmarshal(body, &resp); err != nil {
@@ -177,7 +180,11 @@ func emitRecoveryTSV(out io.Writer, body []byte) error {
 	if d.Action == "" {
 		d.Action = "none"
 	}
-	if _, err := fmt.Fprintf(out, "%d\t%s\t%s\n", d.Epoch, d.Action, d.Snapshot); err != nil {
+	required := "0"
+	if d.SnapshotRequired {
+		required = "1"
+	}
+	if _, err := fmt.Fprintf(out, "%d\t%s\t%s\t%s\n", d.Epoch, d.Action, d.Snapshot, required); err != nil {
 		return err
 	}
 	return nil

cmd/spinifex/cmd/admin.go

@@ -1180,6 +1180,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		externalPools = append(externalPools, admin.PoolData{
 			Name: host.NATTransitPoolName, Gateway: host.NATTransitGatewayIP,
 			PrefixLen: 24, DNSServers: dnsServers,
+			GwLrpRangeStart: host.NATTransitGwLrpStart, GwLrpRangeEnd: host.NATTransitGwLrpEnd,
 		})
 		if natPublicPool {
 			// WiFi/WWAN uplinks drop frames with foreign source MACs, so DHCP
@@ -1266,66 +1267,91 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}
 	fmt.Printf("✅ Created config directory: %s\n", configDir)
 
-	// Generate system credentials (for service-to-service auth in config files)
-	accessKey, err := admin.GenerateAWSAccessKey()
+	// Identity and crypto material is load-or-generate: a fresh install mints a
+	// new identity bundle, but a --force re-init preserves the existing one so
+	// data sealed under it (NATS KV secrets, sealed fragments, encrypted volumes)
+	// stays decryptable.
+	masterKey, masterKeyExisted, err := ensureMasterKey(configDir)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error generating access key: %v\n", err)
-		os.Exit(1)
-	}
-	secretKey, err := admin.GenerateAWSSecretKey()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error generating secret key: %v\n", err)
+		fmt.Fprintf(os.Stderr, "Error preparing IAM master key: %v\n", err)
 		os.Exit(1)
 	}
 	accountID := admin.SystemAccountID()
-
-	// Generate IAM master key (AES-256, used to encrypt secrets in NATS KV)
-	masterKey, err := handlers_iam.GenerateMasterKey()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error generating IAM master key: %v\n", err)
-		os.Exit(1)
-	}
 	bootstrapDir := filepath.Join(spxRoot, "awsgw")
-	bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error writing bootstrap files: %v\n", err)
-		os.Exit(1)
-	}
-	if err := writeSystemCredentials(configDir, accessKey, secretKey); err != nil {
-		fmt.Fprintf(os.Stderr, "Error writing system credentials: %v\n", err)
-		os.Exit(1)
-	}
-	fmt.Println("\n🔐 Generated IAM master key")
-	fmt.Printf("   Master key: %s\n", filepath.Join(configDir, "master.key"))
-	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
-	fmt.Printf("   System creds: %s\n", filepath.Join(configDir, "system-credentials.json"))
 
-	// Predastore encryption key is per-node and never transmitted; generate
-	// it locally now so the service has it on first start.
+	var accessKey, secretKey, adminAccessKey, adminSecretKey string
+	if masterKeyExisted {
+		// Preserve path: reuse the existing identity. The system credentials must
+		// match what seeded the NATS KV `system` secret, so load them rather than
+		// mint new ones. The admin credentials are not recovered: bootstrap.json is
+		// consumed and deleted by awsgw after first boot, and the operator's copy
+		// already lives in ~/.aws/credentials. Leaving them empty makes
+		// finalizeNodeSetup refresh only ~/.aws/config (endpoint/CA for a changed
+		// bind IP), not the credentials.
+		accessKey, secretKey, err = loadSystemCredentials(configDir)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error loading preserved system credentials: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("\n🔐 Preserved existing identity (master key, credentials and CA unchanged)")
+		fmt.Printf("   Master key: %s\n", filepath.Join(configDir, "master.key"))
+	} else {
+		// Fresh install: mint system + admin credentials and seed the bootstrap files.
+		accessKey, err = admin.GenerateAWSAccessKey()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error generating access key: %v\n", err)
+			os.Exit(1)
+		}
+		secretKey, err = admin.GenerateAWSSecretKey()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error generating secret key: %v\n", err)
+			os.Exit(1)
+		}
+		bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error writing bootstrap files: %v\n", err)
+			os.Exit(1)
+		}
+		if err := writeSystemCredentials(configDir, accessKey, secretKey); err != nil {
+			fmt.Fprintf(os.Stderr, "Error writing system credentials: %v\n", err)
+			os.Exit(1)
+		}
+		adminAccessKey = bootstrapResult.AdminAccessKey
+		adminSecretKey = bootstrapResult.AdminSecretKey
+		fmt.Println("\n🔐 Generated IAM master key")
+		fmt.Printf("   Master key: %s\n", filepath.Join(configDir, "master.key"))
+		fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
+		fmt.Printf("   System creds: %s\n", filepath.Join(configDir, "system-credentials.json"))
+	}
+
+	// Predastore encryption key is per-node and never transmitted; load-or-generate
+	// so the service has it on first start and a re-init keeps sealed fragments.
 	predastoreKeyPath, err := writePredastoreEncryptionKey(configDir)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error generating predastore encryption key: %v\n", err)
+		fmt.Fprintf(os.Stderr, "Error preparing predastore encryption key: %v\n", err)
 		os.Exit(1)
 	}
-	fmt.Println("\n🔐 Generated predastore encryption key (per-node, never transmitted)")
+	fmt.Println("\n🔐 Predastore encryption key ready (per-node, never transmitted)")
 	fmt.Printf("   Key: %s\n", predastoreKeyPath)
 
-	// Viperblock at-rest encryption key is cluster-wide; on a single-node init
-	// there are no joiners, so generate it locally and enable encryption by
-	// default for all volumes created on this install.
-	viperblockKeyPath, err := writeViperblockEncryptionKey(configDir)
+	// Viperblock at-rest encryption key is cluster-wide; load-or-generate so a
+	// re-init keeps the existing key (and its sealed volumes). The bytes feed the
+	// multi-node leader's key distribution below.
+	viperblockKey, viperblockKeyPath, err := ensureViperblockEncryptionKey(configDir)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error generating viperblock encryption key: %v\n", err)
+		fmt.Fprintf(os.Stderr, "Error preparing viperblock encryption key: %v\n", err)
 		os.Exit(1)
 	}
-	fmt.Println("\n🔐 Generated viperblock at-rest encryption key")
+	fmt.Println("\n🔐 Viperblock at-rest encryption key ready")
 	fmt.Printf("   Key: %s\n", viperblockKeyPath)
 
-	fmt.Printf("\n🔑 Generated admin credentials (save these — they won't be shown again):\n")
-	fmt.Printf("   Access Key:  %s\n", bootstrapResult.AdminAccessKey)
-	fmt.Printf("   Secret Key:  %s\n", bootstrapResult.AdminSecretKey)
-	fmt.Printf("   Account:     %s (%s)\n", admin.DefaultAccountName(), admin.DefaultAccountID())
-	fmt.Printf("   AWS Profile: spinifex\n")
+	if !masterKeyExisted {
+		fmt.Printf("\n🔑 Generated admin credentials (save these — they won't be shown again):\n")
+		fmt.Printf("   Access Key:  %s\n", adminAccessKey)
+		fmt.Printf("   Secret Key:  %s\n", adminSecretKey)
+		fmt.Printf("   Account:     %s (%s)\n", admin.DefaultAccountName(), admin.DefaultAccountID())
+		fmt.Printf("   AWS Profile: spinifex\n")
+	}
 
 	// Generate SSL certificates (with bind IP in SANs for multi-node support)
 	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP, region, config.DefaultAWSInternalSuffix)
@@ -1391,7 +1417,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 			networkConfig.BootstrapSubnetCidr = handlers_ec2_vpc.DefaultSubnetCidr
 		}
 
-		runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, natsToken, clusterName,
+		runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey,
+			masterKey, viperblockKey, natsToken, clusterName,
 			configDir, spxRoot, certPath, region, az, node, bindIP, advertiseIP, clusterBind, email,
 			port, nodes, formationTimeoutStr, tokenTTLStr, services, networkConfig)
 		return
@@ -1543,7 +1570,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	finalizeNodeSetup(spxRoot, certPath, bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, bindIP, advertiseIP)
+	finalizeNodeSetup(spxRoot, certPath, adminAccessKey, adminSecretKey, region, bindIP)
 
 	// Write node.conf so spx admin banner works on source installs (not just ISO).
 	nodeHostname, _ := os.Hostname()
@@ -1574,7 +1601,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 // runAdminInitMultiNode handles the multi-node formation path for admin init.
 // It starts a formation server, registers this node, waits for all nodes to join,
 // then generates configs with complete cluster topology.
-func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, natsToken, clusterName,
+func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey string,
+	masterKey, viperblockKey []byte, natsToken, clusterName,
 	configDir, spxRoot, certPath, region, az, node, bindIP, advertiseIP, clusterBind, email string,
 	port, expectedNodes int, formationTimeoutStr, tokenTTLStr string, services []string, networkConfig *formation.NetworkConfig) {
 	formationTimeout, err := time.ParseDuration(formationTimeoutStr)
@@ -1601,47 +1629,11 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}
 
-	// Generate IAM master key for the cluster
-	masterKey, err := handlers_iam.GenerateMasterKey()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error generating IAM master key: %v\n", err)
-		os.Exit(1)
-	}
-	bootstrapDir := filepath.Join(spxRoot, "awsgw")
-	bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error writing bootstrap files: %v\n", err)
-		os.Exit(1)
-	}
-	fmt.Println("\n🔐 Generated IAM master key")
-	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
-
-	// Predastore encryption key is per-node and never distributed via the
-	// formation server. Generate only the leader's own key here; each
-	// joiner generates its own during `spx admin join`.
-	predastoreKeyPath, err := writePredastoreEncryptionKey(configDir)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error generating predastore encryption key: %v\n", err)
-		os.Exit(1)
-	}
-	fmt.Println("\n🔐 Generated predastore encryption key (per-node, never transmitted)")
-	fmt.Printf("   Key: %s\n", predastoreKeyPath)
-
-	// Viperblock at-rest encryption key is cluster-wide and shared: the leader
-	// generates it once and distributes it to joiners via the formation server
-	// so a volume sealed on any node can be opened on any other.
-	viperblockKey, err := handlers_iam.GenerateMasterKey()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error generating viperblock encryption key: %v\n", err)
-		os.Exit(1)
-	}
-	viperblockKeyPath, err := saveViperblockEncryptionKey(configDir, viperblockKey)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error saving viperblock encryption key: %v\n", err)
-		os.Exit(1)
-	}
-	fmt.Println("\n🔐 Generated viperblock at-rest encryption key (cluster-wide, shared with joiners)")
-	fmt.Printf("   Key: %s\n", viperblockKeyPath)
+	// Identity, predastore and viperblock keys were prepared load-or-generate by
+	// the caller: reuse them here so a re-init never rotates cluster crypto and
+	// the leader distributes the same viperblock key to joiners. The path is
+	// deterministic and already written on disk.
+	viperblockKeyPath := filepath.Join(configDir, "viperblock", "encryption.key")
 
 	// Read CA cert/key for distribution to joining nodes
 	caCertData, err := os.ReadFile(filepath.Join(configDir, "ca.pem"))
@@ -1662,8 +1654,8 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		NatsToken:      natsToken,
 		ClusterName:    clusterName,
 		Region:         region,
-		AdminAccessKey: bootstrapResult.AdminAccessKey,
-		AdminSecretKey: bootstrapResult.AdminSecretKey,
+		AdminAccessKey: adminAccessKey,
+		AdminSecretKey: adminSecretKey,
 	}
 
 	fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData), networkConfig, joinToken, tokenTTL)
@@ -1800,7 +1792,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}
 
-	finalizeNodeSetup(spxRoot, certPath, bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, bindIP, advertiseIP)
+	finalizeNodeSetup(spxRoot, certPath, adminAccessKey, adminSecretKey, region, bindIP)
 
 	// Keep formation server running briefly so joining nodes can fetch complete status
 	fmt.Println("\n⏳ Waiting for joining nodes to fetch cluster data...")
@@ -2257,7 +2249,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	finalizeNodeSetup(dataDir, caCertPath, creds.AdminAccessKey, creds.AdminSecretKey, creds.Region, bindIP, advertiseIP)
+	finalizeNodeSetup(dataDir, caCertPath, creds.AdminAccessKey, creds.AdminSecretKey, creds.Region, bindIP)
 
 	// Print cluster summary
 	fmt.Println("\n🎉 Node successfully joined cluster!")
@@ -2567,12 +2559,10 @@ func generateAndWriteConfigs(dirs configDirs, spinifexTomlPath string, settings
 }
 
 // finalizeNodeSetup configures AWS credentials, creates service directories,
-// and sets ownership when running as root. advertiseIP is the off-host dial
-// target for this node; it is threaded through SetupAWSCredentials's wanIP
-// param for a future --operator-endpoint flag (see SetupAWSCredentials doc).
-func finalizeNodeSetup(dataDir, certPath, adminAccessKey, adminSecretKey, region, bindIP, advertiseIP string) {
+// and sets ownership when running as root.
+func finalizeNodeSetup(dataDir, certPath, adminAccessKey, adminSecretKey, region, bindIP string) {
 	fmt.Println("\n🔧 Configuring AWS credentials...")
-	if err := admin.SetupAWSCredentials(adminAccessKey, adminSecretKey, region, certPath, bindIP, advertiseIP); err != nil {
+	if err := admin.SetupAWSCredentials(adminAccessKey, adminSecretKey, region, certPath, bindIP); err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err)
 	} else {
 		fmt.Println("✅ AWS credentials configured")
@@ -2626,6 +2616,48 @@ type writeBootstrapResult struct {
 	AdminSecretKey string
 }
 
+// ensureMasterKey load-or-generates the IAM master key at <configDir>/master.key.
+// It returns the key bytes and whether the key already existed on disk. On a
+// --force re-init the existing key is preserved: rotating it would orphan every
+// IAM secret in NATS KV (e.g. the ECR signing key) that was encrypted under it.
+func ensureMasterKey(configDir string) (key []byte, existed bool, err error) {
+	keyPath := filepath.Join(configDir, "master.key")
+	if admin.FileExists(keyPath) {
+		key, err = handlers_iam.LoadMasterKey(keyPath)
+		if err != nil {
+			return nil, false, fmt.Errorf("load master key: %w", err)
+		}
+		return key, true, nil
+	}
+	key, err = handlers_iam.GenerateMasterKey()
+	if err != nil {
+		return nil, false, fmt.Errorf("generate master key: %w", err)
+	}
+	return key, false, nil
+}
+
+// loadSystemCredentials reads the preserved system access/secret key from
+// <configDir>/system-credentials.json. On a re-init the configs must render the
+// same system credentials that seeded the NATS KV `system` secret, else SigV4
+// auth between services fails.
+func loadSystemCredentials(configDir string) (accessKey, secretKey string, err error) {
+	data, err := os.ReadFile(filepath.Join(configDir, "system-credentials.json"))
+	if err != nil {
+		return "", "", fmt.Errorf("read system credentials: %w", err)
+	}
+	var creds struct {
+		AccessKeyID     string `json:"access_key_id"`
+		SecretAccessKey string `json:"secret_access_key"`
+	}
+	if err := json.Unmarshal(data, &creds); err != nil {
+		return "", "", fmt.Errorf("parse system credentials: %w", err)
+	}
+	if creds.AccessKeyID == "" || creds.SecretAccessKey == "" {
+		return "", "", fmt.Errorf("system credentials missing access/secret key")
+	}
+	return creds.AccessKeyID, creds.SecretAccessKey, nil
+}
+
 // writeBootstrapFiles generates new admin credentials and writes the bootstrap
 // files (master.key to configDir, bootstrap.json to bootstrapDir).
 // Used by init flows (single and multi-node).
@@ -2647,47 +2679,57 @@ func writeBootstrapFiles(configDir, bootstrapDir string, masterKey []byte, acces
 	}, nil
 }
 
-// writePredastoreEncryptionKey generates a fresh 32-byte AES-256 master key
-// for this node's predastore data dir and writes it to
-// <configDir>/predastore/encryption.key with mode 0600. The key is per-node:
-// every node generates its own at init/join time and never transmits it,
-// because predastore only opens fragments on the node that sealed them.
+// writePredastoreEncryptionKey load-or-generates this node's per-node predastore
+// key at <configDir>/predastore/encryption.key (mode 0600). The key is per-node:
+// every node generates its own at init/join time and never transmits it, because
+// predastore only opens fragments on the node that sealed them. An existing key
+// is preserved so a --force re-init does not orphan already-sealed fragments.
 func writePredastoreEncryptionKey(configDir string) (string, error) {
 	predastoreDir := filepath.Join(configDir, "predastore")
 	if err := os.MkdirAll(predastoreDir, 0750); err != nil {
 		return "", fmt.Errorf("create predastore config dir: %w", err)
 	}
+	keyPath := filepath.Join(predastoreDir, "encryption.key")
+	if admin.FileExists(keyPath) {
+		return keyPath, nil
+	}
 	key, err := handlers_iam.GenerateMasterKey()
 	if err != nil {
 		return "", fmt.Errorf("generate predastore encryption key: %w", err)
 	}
-	keyPath := filepath.Join(predastoreDir, "encryption.key")
 	if err := handlers_iam.SaveMasterKey(keyPath, key); err != nil {
 		return "", fmt.Errorf("save predastore encryption key: %w", err)
 	}
 	return keyPath, nil
 }
 
-// writeViperblockEncryptionKey generates a fresh 32-byte AES-256 master key for
-// viperblock at-rest encryption and writes it to
-// <configDir>/viperblock/encryption.key with mode 0600. Unlike the predastore
-// key, this key is cluster-wide and shared: the leader generates it once at init
-// and distributes it to joiners via the formation server, so a volume sealed on
-// any node can be opened on any other. Loaded at startup via masterkey.LoadShared.
-func writeViperblockEncryptionKey(configDir string) (string, error) {
+// ensureViperblockEncryptionKey load-or-generates the cluster-wide viperblock
+// at-rest key at <configDir>/viperblock/encryption.key (mode 0600) and returns
+// its bytes and path. Unlike the predastore key it is shared: the leader loads
+// these bytes to distribute the same key to joiners via the formation server, so
+// a volume sealed on any node can be opened on any other. An existing key is
+// preserved so a --force re-init does not orphan already-encrypted volumes.
+func ensureViperblockEncryptionKey(configDir string) ([]byte, string, error) {
 	viperblockDir := filepath.Join(configDir, "viperblock")
 	if err := os.MkdirAll(viperblockDir, 0750); err != nil {
-		return "", fmt.Errorf("create viperblock config dir: %w", err)
+		return nil, "", fmt.Errorf("create viperblock config dir: %w", err)
+	}
+	keyPath := filepath.Join(viperblockDir, "encryption.key")
+	if admin.FileExists(keyPath) {
+		key, err := handlers_iam.LoadMasterKey(keyPath)
+		if err != nil {
+			return nil, "", fmt.Errorf("load viperblock encryption key: %w", err)
+		}
+		return key, keyPath, nil
 	}
 	key, err := handlers_iam.GenerateMasterKey()
 	if err != nil {
-		return "", fmt.Errorf("generate viperblock encryption key: %w", err)
+		return nil, "", fmt.Errorf("generate viperblock encryption key: %w", err)
 	}
-	keyPath := filepath.Join(viperblockDir, "encryption.key")
 	if err := handlers_iam.SaveMasterKey(keyPath, key); err != nil {
-		return "", fmt.Errorf("save viperblock encryption key: %w", err)
+		return nil, "", fmt.Errorf("save viperblock encryption key: %w", err)
 	}
-	return keyPath, nil
+	return key, keyPath, nil
 }
 
 // saveViperblockEncryptionKey writes an already-generated 32-byte viperblock

cmd/spinifex/cmd/admin_eks.go

@@ -0,0 +1,77 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
+	"github.com/spf13/cobra"
+)
+
+var adminEksCmd = &cobra.Command{
+	Use:   "eks",
+	Short: "Manage EKS control-plane disaster recovery",
+}
+
+var adminEksRestoreSnapshotCmd = &cobra.Command{
+	Use:   "restore-snapshot",
+	Short: "Rebuild a single-CP cluster's control plane from an etcd snapshot",
+	Long: `restore-snapshot drives the single-CP total-loss DR path: it launches a fresh
+control-plane VM as a cluster-init seed, directs its boot-time recovery agent to
+run k3s server --cluster-reset (restoring the given etcd snapshot, or the latest
+one found in predastore when --snapshot is omitted), and re-points the cluster
+NLB target groups at the new CP.
+
+Only single-CP clusters are supported. An HA cluster has a potentially
+surviving etcd quorum and should be recovered via quorum reformation instead
+of this destructive single-node rebuild.`,
+	Run: runAdminEksRestoreSnapshot,
+}
+
+func init() {
+	adminCmd.AddCommand(adminEksCmd)
+	adminEksCmd.AddCommand(adminEksRestoreSnapshotCmd)
+
+	adminEksRestoreSnapshotCmd.Flags().String("cluster", "", "Cluster name to restore (required)")
+	adminEksRestoreSnapshotCmd.Flags().String("snapshot", "", "Etcd snapshot key to restore; defaults to the latest snapshot found in predastore")
+	adminEksRestoreSnapshotCmd.Flags().String("account", "", "Account ID that owns the cluster; defaults to the bootstrap account")
+	_ = adminEksRestoreSnapshotCmd.MarkFlagRequired("cluster")
+}
+
+func runAdminEksRestoreSnapshot(cmd *cobra.Command, _ []string) {
+	clusterName, _ := cmd.Flags().GetString("cluster")
+	snapshot, _ := cmd.Flags().GetString("snapshot")
+	accountID, _ := cmd.Flags().GetString("account")
+	if accountID == "" {
+		accountID = admin.DefaultAccountID()
+	}
+
+	_, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	svc := handlers_eks.NewNATSEKSService(nc)
+	// The server side launches a VM (30-60s) then fences the old CP with retries,
+	// so the RPC needs generous headroom over a bare launch.
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second)
+	defer cancel()
+
+	fmt.Printf("Restoring control plane for cluster %q (account %s)...\n", clusterName, accountID)
+	out, err := svc.RestoreSnapshot(ctx, &handlers_eks.RestoreSnapshotInput{
+		ClusterName: clusterName,
+		Snapshot:    snapshot,
+	}, accountID)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("New control plane %s launched (snapshot %s)\n", out.NewInstanceID, out.Snapshot)
+	fmt.Printf("Status: %s\n", out.Status)
+}

cmd/spinifex/cmd/admin_nat_test.go

@@ -46,6 +46,55 @@ func TestSpinifexTomlTemplate_NATMode(t *testing.T) {
 	assert.NotContains(t, content, "range_start", "nat mode has no public IP range")
 }
 
+func TestSpinifexTomlTemplate_GwLrpRange(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node: "node1", Az: "ap-southeast-2a", Port: "4432",
+		Region: "ap-southeast-2", BindIP: "10.11.12.1",
+		AccessKey: "AKIATEST", SecretKey: "SECRET", AccountID: "123456789012",
+		NatsToken: "token", ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641", OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "nat", BridgeMode: "nat",
+		Pools: []admin.PoolData{{
+			Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24,
+			GwLrpRangeStart: "100.127.0.16", GwLrpRangeEnd: "100.127.0.254",
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `gw_lrp_range_start = "100.127.0.16"`)
+	assert.Contains(t, content, `gw_lrp_range_end   = "100.127.0.254"`)
+}
+
+func TestSpinifexTomlTemplate_GwLrpRangeOmittedWhenUnset(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node: "node1", Az: "ap-southeast-2a", Port: "4432",
+		Region: "ap-southeast-2", BindIP: "10.11.12.1",
+		AccessKey: "AKIATEST", SecretKey: "SECRET", AccountID: "123456789012",
+		NatsToken: "token", ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641", OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "pool", ExternalIface: "enp0s3",
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "static",
+			Start: "192.168.1.150", End: "192.168.1.250",
+			Gateway: "192.168.1.1", PrefixLen: 24,
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	assert.NotContains(t, string(data), "gw_lrp_range", "unset range must not render")
+}
+
 func TestSpinifexTomlTemplate_PoolModeOmitsBridgeMode(t *testing.T) {
 	dir := t.TempDir()
 	path := filepath.Join(dir, "spinifex.toml")

cmd/spinifex/cmd/admin_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/formation"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	toml "github.com/pelletier/go-toml/v2"
 	"github.com/spf13/cobra"
@@ -255,23 +256,6 @@ func TestWritePredastoreEncryptionKey(t *testing.T) {
 	assert.NotEqual(t, key1, key2, "per-node keys must differ")
 }
 
-// The viperblock key gates at-rest encryption for every volume on a fresh
-// install. masterkey.LoadShared rejects anything that isn't exactly 32 bytes at
-// 0640-or-stricter, so a regression in size/mode would silently fall back to
-// cleartext (empty path) or fail service startup.
-func TestWriteViperblockEncryptionKey(t *testing.T) {
-	configDir := t.TempDir()
-
-	keyPath, err := writeViperblockEncryptionKey(configDir)
-	require.NoError(t, err)
-	assert.Equal(t, filepath.Join(configDir, "viperblock", "encryption.key"), keyPath)
-
-	info, err := os.Stat(keyPath)
-	require.NoError(t, err)
-	assert.Equal(t, int64(32), info.Size(), "viperblock master key must be exactly 32 bytes")
-	assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "viperblock master key must be mode 0600")
-}
-
 // Joiners persist the leader's shared key via saveViperblockEncryptionKey
 // against a configDir with no viperblock/ subdir yet. The bytes round-trip
 // verbatim — the whole cluster shares one key, so any mutation would orphan
@@ -292,6 +276,181 @@ func TestSaveViperblockEncryptionKey_RoundTrip(t *testing.T) {
 	assert.Equal(t, key, got, "shared key must be written verbatim")
 }
 
+// A re-init must not rotate the per-node predastore key: rotating it would orphan
+// every fragment already sealed under the old key. The helper preserves an
+// existing key byte-for-byte.
+func TestWritePredastoreEncryptionKey_PreservesExisting(t *testing.T) {
+	configDir := t.TempDir()
+
+	keyPath, err := writePredastoreEncryptionKey(configDir)
+	require.NoError(t, err)
+	orig, err := os.ReadFile(keyPath)
+	require.NoError(t, err)
+
+	keyPath2, err := writePredastoreEncryptionKey(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, keyPath, keyPath2)
+	after, err := os.ReadFile(keyPath2)
+	require.NoError(t, err)
+	assert.Equal(t, orig, after, "existing predastore key must be preserved on re-init")
+}
+
+// ensureViperblockEncryptionKey generates a shared 32-byte 0600 key on a fresh
+// dir and returns bytes that match the on-disk file; a second call preserves it
+// so a re-init keeps every encrypted volume readable.
+func TestEnsureViperblockEncryptionKey(t *testing.T) {
+	configDir := t.TempDir()
+
+	key, keyPath, err := ensureViperblockEncryptionKey(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, filepath.Join(configDir, "viperblock", "encryption.key"), keyPath)
+	assert.Len(t, key, 32, "viperblock key must be 32 bytes")
+
+	info, err := os.Stat(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, int64(32), info.Size(), "viperblock master key must be exactly 32 bytes")
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "viperblock master key must be mode 0600")
+
+	onDisk, err := os.ReadFile(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, key, onDisk, "returned bytes must match the file")
+
+	key2, _, err := ensureViperblockEncryptionKey(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, key, key2, "existing viperblock key must be preserved on re-init")
+}
+
+// A present-but-corrupt viperblock key must fail loud rather than regenerate,
+// which would orphan every already-encrypted volume in the cluster.
+func TestEnsureViperblockEncryptionKey_CorruptFailsLoud(t *testing.T) {
+	configDir := t.TempDir()
+	keyDir := filepath.Join(configDir, "viperblock")
+	require.NoError(t, os.MkdirAll(keyDir, 0750))
+	keyPath := filepath.Join(keyDir, "encryption.key")
+	require.NoError(t, os.WriteFile(keyPath, []byte("bad"), 0600))
+
+	_, _, err := ensureViperblockEncryptionKey(configDir)
+	require.Error(t, err, "a truncated viperblock key must error, not regenerate")
+
+	after, err := os.ReadFile(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, []byte("bad"), after, "corrupt viperblock key must not be overwritten")
+}
+
+// ensureMasterKey generates a fresh in-memory key without touching disk (the
+// bootstrap writer persists it), and preserves an existing on-disk key so a
+// re-init never rotates the key that encrypts IAM secrets in NATS KV.
+func TestEnsureMasterKey(t *testing.T) {
+	t.Run("FreshGeneratesWithoutWriting", func(t *testing.T) {
+		configDir := t.TempDir()
+		key, existed, err := ensureMasterKey(configDir)
+		require.NoError(t, err)
+		assert.False(t, existed, "no master.key present yet")
+		assert.Len(t, key, 32)
+		assert.NoFileExists(t, filepath.Join(configDir, "master.key"),
+			"ensureMasterKey must not persist on the fresh path")
+	})
+
+	t.Run("PreservesExisting", func(t *testing.T) {
+		configDir := t.TempDir()
+		seed := make([]byte, 32)
+		for i := range seed {
+			seed[i] = byte(200 - i)
+		}
+		require.NoError(t, handlers_iam.SaveMasterKey(filepath.Join(configDir, "master.key"), seed))
+
+		key, existed, err := ensureMasterKey(configDir)
+		require.NoError(t, err)
+		assert.True(t, existed, "existing master.key must be detected")
+		assert.Equal(t, seed, key, "existing master key must be loaded verbatim")
+	})
+
+	// A present-but-corrupt master.key must fail loud, never be silently
+	// regenerated: rotating it would orphan every IAM secret in NATS KV.
+	t.Run("CorruptFailsLoud", func(t *testing.T) {
+		configDir := t.TempDir()
+		keyPath := filepath.Join(configDir, "master.key")
+		require.NoError(t, os.WriteFile(keyPath, []byte("too-short"), 0600))
+
+		_, _, err := ensureMasterKey(configDir)
+		require.Error(t, err, "a truncated master.key must error, not regenerate")
+
+		after, err := os.ReadFile(keyPath)
+		require.NoError(t, err)
+		assert.Equal(t, []byte("too-short"), after, "corrupt master key must not be overwritten")
+	})
+}
+
+// The identity bundle is preserved holistically on re-init: the load helpers
+// recover the exact system credentials and never rewrite bootstrap.json. Admin
+// credentials are not recovered from disk — awsgw consumes and deletes
+// bootstrap.json after first boot, so the preserve path must not read it back.
+func TestPreservedIdentityBundle(t *testing.T) {
+	configDir := t.TempDir()
+	bootstrapDir := filepath.Join(configDir, "awsgw")
+
+	masterKey, err := handlers_iam.GenerateMasterKey()
+	require.NoError(t, err)
+	const (
+		sysAccess   = "AKIASYSTEM0000000000"
+		sysSecret   = "system-secret-value"
+		adminAccess = "AKIAADMIN00000000000"
+		adminSecret = "admin-secret-value"
+		accountID   = "123456789012"
+	)
+	require.NoError(t, writeBootstrapFilesWithAdmin(configDir, bootstrapDir, masterKey,
+		sysAccess, sysSecret, accountID, adminAccess, adminSecret))
+	require.NoError(t, writeSystemCredentials(configDir, sysAccess, sysSecret))
+
+	bootstrapPath := filepath.Join(bootstrapDir, "bootstrap.json")
+	bootstrapBefore, err := os.ReadFile(bootstrapPath)
+	require.NoError(t, err)
+
+	// master.key round-trips and is flagged as pre-existing.
+	loadedKey, existed, err := ensureMasterKey(configDir)
+	require.NoError(t, err)
+	assert.True(t, existed)
+	assert.Equal(t, masterKey, loadedKey)
+
+	// System credentials come back verbatim (they must match the NATS KV seed).
+	gotSysAccess, gotSysSecret, err := loadSystemCredentials(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, sysAccess, gotSysAccess)
+	assert.Equal(t, sysSecret, gotSysSecret)
+
+	// The preserve helpers must not rewrite the seed file.
+	bootstrapAfter, err := os.ReadFile(bootstrapPath)
+	require.NoError(t, err)
+	assert.Equal(t, bootstrapBefore, bootstrapAfter, "bootstrap.json must not be rewritten on re-init")
+}
+
+// loadSystemCredentials surfaces a clear error when the file is absent rather
+// than returning empty credentials that would silently break SigV4 auth.
+func TestLoadSystemCredentials_Missing(t *testing.T) {
+	_, _, err := loadSystemCredentials(t.TempDir())
+	require.Error(t, err)
+}
+
+// A present-but-empty or malformed system-credentials.json must error rather
+// than yield blank credentials that would silently break SigV4 between services.
+func TestLoadSystemCredentials_Invalid(t *testing.T) {
+	t.Run("EmptyValues", func(t *testing.T) {
+		configDir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(configDir, "system-credentials.json"),
+			[]byte(`{"access_key_id":"","secret_access_key":""}`), 0600))
+		_, _, err := loadSystemCredentials(configDir)
+		require.Error(t, err)
+	})
+
+	t.Run("MalformedJSON", func(t *testing.T) {
+		configDir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(configDir, "system-credentials.json"),
+			[]byte("not json"), 0600))
+		_, _, err := loadSystemCredentials(configDir)
+		require.Error(t, err)
+	})
+}
+
 // A fresh install must render encryption_key_file so viperblockd enables
 // at-rest encryption; an empty field (upgrade / legacy) must omit the line
 // entirely so the volume stays in cleartext legacy mode.

cmd/spinifex/cmd/cluster.go

@@ -123,7 +123,7 @@ func runClusterShutdown(cmd *cobra.Command, args []string) {
 		phaseStart := time.Now()
 		fmt.Printf("[%s] Sending to %d node(s)...\n", strings.ToUpper(phase), nodeCount)
 
-		acks, err := collectShutdownACKs(nc, topic, reqData, nodeCount, timeout)
+		acks, err := collectShutdownACKs(nc, topic, reqData, nodeCount, "", timeout)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "[%s] Error: %v\n", strings.ToUpper(phase), err)
 			if !force {
@@ -216,36 +216,14 @@ func runNodeDrainLocal(cmd *cobra.Command, args []string) {
 // from the local node, ignoring ACKs from any other node. Returns an error if
 // the local node does not respond within timeout.
 func collectLocalShutdownACK(nc *nats.Conn, topic string, reqData []byte, node string, timeout time.Duration) (daemon.ShutdownACK, error) {
-	inbox := nats.NewInbox()
-	sub, err := nc.SubscribeSync(inbox)
+	acks, err := collectShutdownACKs(nc, topic, reqData, 1, node, timeout)
 	if err != nil {
-		return daemon.ShutdownACK{}, fmt.Errorf("failed to subscribe to inbox: %w", err)
+		return daemon.ShutdownACK{}, err
 	}
-	defer sub.Unsubscribe()
-
-	if err := nc.PublishRequest(topic, inbox, reqData); err != nil {
-		return daemon.ShutdownACK{}, fmt.Errorf("failed to publish request: %w", err)
-	}
-	nc.Flush()
-
-	deadline := time.Now().Add(timeout)
-	for {
-		remaining := time.Until(deadline)
-		if remaining <= 0 {
-			return daemon.ShutdownACK{}, fmt.Errorf("timeout waiting for local node %q ACK", node)
-		}
-		msg, err := sub.NextMsg(remaining)
-		if err != nil {
-			return daemon.ShutdownACK{}, fmt.Errorf("timeout waiting for local node %q ACK", node)
-		}
-		var ack daemon.ShutdownACK
-		if err := json.Unmarshal(msg.Data, &ack); err != nil {
-			continue
-		}
-		if ack.Node == node {
-			return ack, nil
-		}
+	if len(acks) == 0 {
+		return daemon.ShutdownACK{}, fmt.Errorf("timeout waiting for local node %q ACK", node)
 	}
+	return acks[0], nil
 }
 
 // runClusterDrainDHCP asks every vpcd to DHCPRELEASE all external-pool leases
@@ -322,7 +300,9 @@ func drainDHCPLeases(nc *nats.Conn, timeout time.Duration) (released, responders
 }
 
 // collectShutdownACKs publishes a shutdown request and collects ACKs from nodes.
-func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount int, timeout time.Duration) ([]daemon.ShutdownACK, error) {
+// When nodeFilter is non-empty, only ACKs from that node count toward nodeCount
+// and all others are ignored (used by the single-node local-drain path).
+func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount int, nodeFilter string, timeout time.Duration) ([]daemon.ShutdownACK, error) {
 	inbox := nats.NewInbox()
 	sub, err := nc.SubscribeSync(inbox)
 	if err != nil {
@@ -350,6 +330,9 @@ func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount
 		if err := json.Unmarshal(msg.Data, &ack); err != nil {
 			continue
 		}
+		if nodeFilter != "" && ack.Node != nodeFilter {
+			continue
+		}
 		acks = append(acks, ack)
 	}
 	return acks, nil

cmd/spinifex/cmd/get.go

@@ -138,6 +138,12 @@ func formatRoles(resp types.NodeStatusResponse) string {
 	if resp.NATSRole != "" {
 		roles = append(roles, "nats:"+resp.NATSRole)
 	}
+	if resp.OVNNBRole != "" {
+		roles = append(roles, "ovn-nb:"+resp.OVNNBRole)
+	}
+	if resp.OVNSBRole != "" {
+		roles = append(roles, "ovn-sb:"+resp.OVNSBRole)
+	}
 	if resp.PredastoreRole != "" {
 		roles = append(roles, "predastore:"+resp.PredastoreRole)
 	}

cmd/spinifex/cmd/get_test.go

@@ -38,6 +38,26 @@ func TestFormatRoles(t *testing.T) {
 			resp: types.NodeStatusResponse{NATSRole: "leader", PredastoreRole: "leader"},
 			want: "nats:leader,predastore:leader",
 		},
+		{
+			name: "ovn roles only",
+			resp: types.NodeStatusResponse{OVNNBRole: "leader", OVNSBRole: "follower"},
+			want: "ovn-nb:leader,ovn-sb:follower",
+		},
+		{
+			name: "all four roles ordered",
+			resp: types.NodeStatusResponse{
+				NATSRole:       "leader",
+				OVNNBRole:      "leader",
+				OVNSBRole:      "follower",
+				PredastoreRole: "follower",
+			},
+			want: "nats:leader,ovn-nb:leader,ovn-sb:follower,predastore:follower",
+		},
+		{
+			name: "ovn-nb leader with nats, sb absent",
+			resp: types.NodeStatusResponse{NATSRole: "follower", OVNNBRole: "leader"},
+			want: "nats:follower,ovn-nb:leader",
+		},
 	}
 
 	for _, tt := range tests {

cmd/spinifex/cmd/service.go

@@ -970,11 +970,6 @@ func init() {
 	viper.BindEnv("encryption-key-file", "SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE")
 	viper.BindPFlag("encryption-key-file", predastoreCmd.PersistentFlags().Lookup("encryption-key-file"))
 
-	// Predastore Backend
-	predastoreCmd.PersistentFlags().String("backend", "distributed", "Predastore (S3) backend")
-	viper.BindEnv("backend", "SPINIFEX_PREDASTORE_BACKEND")
-	viper.BindPFlag("backend", predastoreCmd.PersistentFlags().Lookup("backend"))
-
 	// Predastore Node ID. Default -1 is dev mode (launch every configured
 	// QUIC node in-process). Production deployments set this to the node's
 	// real ID (>= 1) via SPINIFEX_PREDASTORE_NODE_ID or --node-id.

cmd/spinifex/cmd/templates/spinifex.toml

@@ -39,6 +39,10 @@ gateway     = "{{.Gateway}}"
 {{- if .GatewayIP}}
 gateway_ip  = "{{.GatewayIP}}"
 {{- end}}
+{{- if .GwLrpRangeStart}}
+gw_lrp_range_start = "{{.GwLrpRangeStart}}"
+gw_lrp_range_end   = "{{.GwLrpRangeEnd}}"
+{{- end}}
 prefix_len  = {{if .PrefixLen}}{{.PrefixLen}}{{else}}24{{end}}
 {{- if .DNSServers}}
 dns_servers = [{{range $i, $s := .DNSServers}}{{if $i}}, {{end}}"{{$s}}"{{end}}]

docs/COMMANDS.md

@@ -54,7 +54,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; add `--external-pool start-end --external-gateway <ip>` or `--external-source=dhcp [--external-bind-bridge <iface>]` for a public pool with full EIP support), `--no-external` | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` |
+| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; add `--external-pool start-end --external-gateway <ip>` or `--external-source=dhcp [--external-bind-bridge <iface>]` for a public pool with full EIP support; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 — they require `pool` mode), `--no-external` | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/`. `--force` on an already-initialized node is idempotent for crypto: it preserves the existing keys (master.key, predastore/viperblock encryption keys), system + admin credentials, and CA, and refreshes only the config files and the CA-signed server certificate (so a changed bind IP / SANs is picked up without breaking already-joined nodes or CA-baked AMIs). A genuine clean slate requires removing the data dirs first. |
 | `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations |
 
 ### Version
@@ -171,6 +171,11 @@ read.
 | `spx admin gpu mig enable` | `--profile <name>` (required, e.g. `1g.10gb`), `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → enables MIG mode on each target (`gpu.EnableMIGMode`) → lists available profiles (`gpu.ListProfiles`) and validates requested profile name → destroys any existing instances (`gpu.DestroyAllInstances`) → creates new instances filling GPU capacity (`gpu.CreateInstances`) → writes `mig_profile` to `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. |
 | `spx admin gpu mig disable` | `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → destroys all GPU instances (`gpu.DestroyAllInstances`) → disables MIG mode (`gpu.DisableMIGMode`) → clears `mig_profile` in `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. |
 
+### EKS Control-Plane Disaster Recovery
+
+| Command | Flags | Description |
+|---------|-------|-------------|
+| `spx admin eks restore-snapshot` | `--cluster` (required), `--snapshot` (optional, defaults to the latest snapshot in predastore), `--account` (optional, defaults to the bootstrap account) | Single-CP total-loss DR path (fail-safe): validates the snapshot exists in predastore BEFORE any mutation (a typo'd/missing key hard-fails, never resets into an empty datastore) → launches a fresh control-plane VM as a cluster-init seed (replaying the persisted create-time launch template) → sets a required-snapshot `RecoveryDirective` (`cluster-reset`) so the boot-time recovery agent aborts rather than resets-into-empty if it cannot fetch the snapshot → persists the replacement in cluster meta BEFORE re-pointing the NLB (so an NLB failure is convergeable by the reconciler, returned as a provisional status, not a hard error) → re-points the cluster NLB's apiserver and konnectivity target groups from the old CP's ENI to the new one → fences the old CP with retries, failing loudly if it cannot be confirmed terminated (split-brain guard). Any failure before the meta commit unwinds the fresh CP (terminate + clear directive) so a re-run does not stack a second resetting control plane. The returned status is provisional — success means the sequence completed, not that etcd is restored and serving; verify cluster health. HA clusters (a spread with a potentially surviving quorum) are rejected — recover those via quorum reformation instead. |
 
 ## AWS-Compatible API
 
@@ -178,7 +183,7 @@ read.
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `run-instances` | `--image-id`, `--instance-type`, `--count`, `--key-name`, `--user-data`, `--subnet-id`, `--security-group-ids`, `--tag-specifications` (instance-scoped), `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination), `--placement` (GroupName), `--iam-instance-profile` (Name/Arn), `--capacity-reservation-specification` (CapacityReservationTarget.CapacityReservationId, targeted-by-id only), `--metadata-options` (HttpPutResponseHopLimit; IMDSv2-only enforced — rejects `http-tokens=optional`) | `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--launch-template`, `--hibernate-options` | **DONE** |
+| `run-instances` | `--image-id`, `--instance-type`, `--count`, `--key-name`, `--user-data`, `--subnet-id`, `--security-group-ids`, `--tag-specifications` (instance-scoped), `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination), `--placement` (GroupName), `--iam-instance-profile` (Name/Arn), `--capacity-reservation-specification` (CapacityReservationTarget.CapacityReservationId, targeted-by-id only), `--metadata-options` (HttpPutResponseHopLimit; IMDSv2-only enforced — rejects `http-tokens=optional`), `--launch-template` (LaunchTemplateId/LaunchTemplateName, Version — resolves `$Default`/`$Latest`; direct params override the template) | `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--hibernate-options` | **DONE** |
 | `describe-instances` | `--instance-ids`, `--filters` (instance-state-name, instance-id, instance-type, vpc-id, subnet-id, tag:*, tag-key, tag-value) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
 | `start-instances` | `--instance-ids` | `--dry-run`, `--force` | **DONE** |
 | `stop-instances` | `--instance-ids` | `--force`, `--hibernate`, `--dry-run` | **DONE** |
@@ -480,11 +485,13 @@ EIP handlers are always registered. Without a public IPAM pool (external mode di
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-launch-template` | — | `--launch-template-name`, `--launch-template-data`, `--tag-specifications` | **NOT STARTED** |
-| `create-launch-template-version` | — | `--launch-template-id`/`--launch-template-name`, `--launch-template-data`, `--source-version` | **NOT STARTED** |
-| `delete-launch-template` | — | `--launch-template-id`/`--launch-template-name`, `--dry-run` | **NOT STARTED** |
-| `describe-launch-templates` | — | `--launch-template-ids`, `--launch-template-names`, `--filters` | **NOT STARTED** |
-| `describe-launch-template-versions` | — | `--launch-template-id`/`--launch-template-name`, `--versions`, `--min-version`, `--max-version` | **NOT STARTED** |
+| `create-launch-template` | `--launch-template-name`, `--launch-template-data` (full nested RequestLaunchTemplateData), `--version-description`, `--tag-specifications` (launch-template-scoped), `--dry-run` (no-op) | `--client-token` (idempotency) | **DONE** |
+| `create-launch-template-version` | `--launch-template-id`/`--launch-template-name`, `--launch-template-data`, `--source-version` (clone-and-override), `--version-description`, `--dry-run` (no-op) | `--client-token`, `--resolve-alias` | **DONE** |
+| `delete-launch-template` | `--launch-template-id`/`--launch-template-name`, `--dry-run` (no-op) | — | **DONE** |
+| `delete-launch-template-versions` | `--launch-template-id`/`--launch-template-name`, `--versions` (rejects the current default version), `--dry-run` (no-op) | — | **DONE** |
+| `modify-launch-template` | `--launch-template-id`/`--launch-template-name`, `--default-version`, `--dry-run` (no-op) | — | **DONE** |
+| `describe-launch-templates` | `--launch-template-ids`, `--launch-template-names`, `--filters` (launch-template-id, launch-template-name, create-time, tag:*, tag-key) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
+| `describe-launch-template-versions` | `--launch-template-id`/`--launch-template-name`, `--versions` (`$Default`/`$Latest`/numeric), `--min-version`, `--max-version`, `--filters` (is-default-version, image-id, instance-type, kernel-id, ram-disk-id, ebs-optimized) | `--max-results`, `--next-token`, `--dry-run`, `--resolve-alias` | **DONE** |
 
 ### EC2 — Dedicated Hosts, IPv4 Pools, DHCP, Capacity Reservations
 
@@ -662,6 +669,12 @@ All IAM operations are account-scoped. Root user (account `000000000000`) bypass
 | `delete-group-policy` | `--group-name`, `--policy-name` | — | **DONE** |
 | `list-group-policies` | `--group-name` | `--max-items`, `--marker` | **DONE** |
 
+### IAM — Account
+
+| Command | Implemented Flags | Missing Flags | Status |
+|---------|-------------------|---------------|--------|
+| `get-account-summary` | — | — | **DONE** |
+
 ---
 
 ## STS

docs/compute/vpc-networking/README.md

@@ -198,6 +198,12 @@ public IPs, no Elastic IPs, and no inbound from WAN — all subnets behave as
 private subnets with internet access. On routed-NAT nodes, adding a public pool
 restores full public IP parity (see below).
 
+> **Limitation (routed-NAT v1):** System instances (ECS/EKS/load-balancer
+> agents) source egress from ExternalIPAM pool IPs, which do not exist in
+> `external_mode=nat`. Those features require `external_mode=pool`. A v2 will
+> either allocate transit IPs for system instances or reject the feature at the
+> API level in nat mode.
+
 The `gateway_ip` is the IP that OVN uses for SNAT. You can set it statically or
 use `setup-ovn.sh --dhcp` to obtain one from the router. This is the router's
 DHCP — not Spinifex's internal OVN DHCP for VMs.
@@ -549,6 +555,7 @@ dns_servers = ["8.8.8.8"]           # DNS for VMs (optional)
 | `region`      | No          | Scopes pool to a region. Instances in this region prefer this pool.                                                                                                    |
 | `az`          | No          | Scopes pool to an AZ. More specific than region.                                                                                                                       |
 | `dns_servers` | No          | DNS servers propagated to VMs via OVN DHCP.                                                                                                                            |
+| `gw_lrp_range_start` / `gw_lrp_range_end` | No | Reserve gateway-LRP IPs for per-VPC OVN routers. When unset, the allocator auto-derives the top 16 host IPs of the pool subnet (~15 concurrent VPCs). Widen to raise the concurrent-VPC ceiling — the `nat-transit` pool defaults to `100.127.0.16`-`100.127.0.254` (239 VPCs). Must not overlap `range_start`/`range_end`. |
 
 ### Why range_start/range_end Instead of CIDR?
 

docs/service-interfaces.yaml

@@ -300,6 +300,14 @@ suites:
     # spinifex_daemon. docker/crane/skopeo data-plane subtests skip when the
     # client binary or registry-host resolution is absent.
 
+  e2e-bedrock:
+    path: tests/e2e/bedrock
+    covers: [awsgw]
+    # Ochre (AWS-Bedrock) control plane: static catalog list/get served
+    # straight from awsgw (gateway/bedrock.go, gateway/bedrock/) — no NATS
+    # hop to spinifex_daemon. Converse (Anthropic-direct / self-host vLLM)
+    # subtests are gated on backend availability and skip by default.
+
   e2e-ecs:
     path: tests/e2e/ecs
     covers: [awsgw, spinifex_daemon, vpcd, viperblockd, predastore]

go.mod

@@ -5,44 +5,48 @@ go 1.26.5
 require (
 	github.com/aws/aws-sdk-go v1.55.8
 	github.com/aws/aws-sdk-go-v2 v1.42.1
-	github.com/aws/aws-sdk-go-v2/config v1.32.27
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.26
+	github.com/aws/aws-sdk-go-v2/config v1.32.29
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.28
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
-	github.com/containerd/containerd/v2 v2.3.2
+	github.com/containerd/containerd/v2 v2.3.3
 	github.com/distribution/reference v0.6.0
-	github.com/go-chi/chi/v5 v5.3.0
+	github.com/go-chi/chi/v5 v5.3.1
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/uuid v1.6.0
 	github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2
 	github.com/klauspost/cpuid/v2 v2.4.0
-	github.com/mulgadc/predastore v1.11.0
-	github.com/mulgadc/viperblock v1.11.0
+	github.com/mulgadc/predastore v1.12.0
+	github.com/mulgadc/viperblock v1.12.0
 	github.com/nats-io/nats-server/v2 v2.14.3
 	github.com/nats-io/nats.go v1.52.0
 	github.com/opencontainers/runtime-spec v1.3.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
-	github.com/pelletier/go-toml/v2 v2.4.2
+	github.com/pelletier/go-toml/v2 v2.4.3
 	github.com/pterm/pterm v0.12.83
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
+	go.opentelemetry.io/contrib/bridges/otelslog v0.19.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0
 	go.opentelemetry.io/contrib/instrumentation/runtime v0.69.0
 	go.opentelemetry.io/otel v1.44.0
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0
+	go.opentelemetry.io/otel/log v0.20.0
 	go.opentelemetry.io/otel/metric v1.44.0
 	go.opentelemetry.io/otel/sdk v1.44.0
+	go.opentelemetry.io/otel/sdk/log v0.20.0
 	go.opentelemetry.io/otel/sdk/metric v1.44.0
 	go.opentelemetry.io/otel/trace v1.44.0
 	go.uber.org/goleak v1.3.0
 	go.yaml.in/yaml/v3 v3.0.4
-	golang.org/x/crypto v0.53.0
-	golang.org/x/sync v0.21.0
-	golang.org/x/sys v0.46.0
+	golang.org/x/crypto v0.54.0
+	golang.org/x/sync v0.22.0
+	golang.org/x/sys v0.47.0
 	gopkg.in/ini.v1 v1.67.3
 )
 
@@ -61,10 +65,10 @@ require (
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.2.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.31.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.43.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.4.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.32.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.44.0 // indirect
 	github.com/aws/smithy-go v1.27.3 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -94,7 +98,7 @@ require (
 	github.com/containerd/ttrpc v1.2.8 // indirect
 	github.com/containerd/typeurl/v2 v2.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/dgraph-io/badger/v4 v4.9.2 // indirect
+	github.com/dgraph-io/badger/v4 v4.9.4 // indirect
 	github.com/dgraph-io/ristretto/v2 v2.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
@@ -139,6 +143,7 @@ require (
 	github.com/minio/crc64nvme v1.1.1 // indirect
 	github.com/minio/highwayhash v1.0.4 // indirect
 	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/sys/capability v0.4.0 // indirect
 	github.com/moby/sys/mountinfo v0.7.2 // indirect
 	github.com/moby/sys/sequential v0.6.0 // indirect
 	github.com/moby/sys/signal v0.7.1 // indirect
@@ -153,6 +158,7 @@ require (
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -176,19 +182,22 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
-	golang.org/x/mod v0.36.0 // indirect
+	golang.org/x/mod v0.37.0 // indirect
 	golang.org/x/net v0.56.0 // indirect
-	golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 // indirect
-	golang.org/x/term v0.44.0 // indirect
-	golang.org/x/text v0.38.0 // indirect
+	golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect
+	golang.org/x/term v0.45.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
-	golang.org/x/tools v0.45.0 // indirect
+	golang.org/x/tools v0.47.0 // indirect
 	golang.org/x/vuln v1.1.4 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260618152121-87f3d3e198d3 // indirect
 	google.golang.org/grpc v1.81.1 // indirect
 	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
+	tags.cncf.io/container-device-interface v1.1.0 // indirect
+	tags.cncf.io/container-device-interface/specs-go v1.1.0 // indirect
 )
 
 // go.opencensus.io -> grpc@v1.33.2 drags in the pre-split monolithic genproto,

go.sum

@@ -402,6 +402,8 @@ codeberg.org/go-latex/latex v0.0.1/go.mod h1:AiC91vVG2uURZRd4ZN1j3mAac0XBrLsxK6+
 codeberg.org/go-latex/latex v0.1.0/go.mod h1:LA0q/AyWIYrqVd+A9Upkgsb+IqPcmSTKc9Dny04MHMw=
 codeberg.org/go-pdf/fpdf v0.10.0/go.mod h1:Y0DGRAdZ0OmnZPvjbMp/1bYxmIPxm0ws4tfoPOc4LjU=
 contrib.go.opencensus.io/exporter/stackdriver v0.13.15-0.20230702191903-2de6d2748484/go.mod h1:uxw+4/0SiKbbVSD/F2tk5pJTdVcfIBBcsQ8gwcu4X+E=
+cyphar.com/go-pathrs v0.2.1 h1:9nx1vOgwVvX1mNBWDu93+vaceedpbsDqo+XuBGL40b8=
+cyphar.com/go-pathrs v0.2.1/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20221208032759-85de2813cf6b/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
@@ -529,13 +531,13 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDk
 github.com/aws/aws-sdk-go-v2/config v1.18.14/go.mod h1:0pI6JQBHKwd0JnwAZS3VCapLKMO++UL2BOkWwyyzTnA=
 github.com/aws/aws-sdk-go-v2/config v1.27.4/go.mod h1:zq2FFXK3A416kiukwpsd+rD4ny6JC7QSkp4QdN1Mp2g=
 github.com/aws/aws-sdk-go-v2/config v1.29.12/go.mod h1:xse1YTjmORlb/6fhkWi8qJh3cvZi4JoVNhc+NbJt4kI=
-github.com/aws/aws-sdk-go-v2/config v1.32.27 h1:SJwJ9Q4kM7v5QVSYYyXj3znRr6lNyZEhSgAXmXXcVbI=
-github.com/aws/aws-sdk-go-v2/config v1.32.27/go.mod h1:uBfrzTRedDmB2u+b6+UlaKJy2O6VSH5un2jP24t/KvQ=
+github.com/aws/aws-sdk-go-v2/config v1.32.29 h1:BcMHHnpiWKogf+gGfpj3K1w+Sktz29XDo/cPSAPO3FU=
+github.com/aws/aws-sdk-go-v2/config v1.32.29/go.mod h1:+Kbhn8Es4kPUph3F/0W7avykytc+Jh2Ld9/msv9ljV4=
 github.com/aws/aws-sdk-go-v2/credentials v1.13.14/go.mod h1:85ckagDuzdIOnZRwws1eLKnymJs3ZM1QwVC1XcuNGOY=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.4/go.mod h1:+30tpwrkOgvkJL1rUZuRLoxcJwtI/OkeBLYnHxJtVe0=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.65/go.mod h1:4zyjAuGOdikpNYiSGpsGz8hLGmUzlY8pc8r9QQ/RXYQ=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.26 h1:Si8kk1kyJnuJWCEgiwpBtTdtgSdR7i611596NnC0YIQ=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.26/go.mod h1:lBckz+W9SAdNtSDw3pYgQUJDJFcBBWry0GSzw+bK0TY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.28 h1:zTXJSsNcoO91/mTXsZoYf0AK8dvNPiA58/VtyGXR+wM=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.28/go.mod h1:Kd9E0JzDBW/q1xbsHFrev/GnbAf5J0Ng8xoyc7HZ91Q=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
@@ -569,25 +571,25 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrK
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31/go.mod h1:I/1+z0VwL1GhQyLgkoHDlygpUZ+iTAwOQ/NsftiUL2I=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.104.2 h1:bAY6O/TDv1HQnvylh9E247IyIKsUWUt2G965S7qX110=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.104.2/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do=
-github.com/aws/aws-sdk-go-v2/service/signin v1.2.2 h1:69JEZSDTQ+UNbTWQJCZMmbpQb5sfc79KUt0O7Pyfjmo=
-github.com/aws/aws-sdk-go-v2/service/signin v1.2.2/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0 h1:XptwLL+UHXgafYMIHTy59IRovLbhz3znkxY2uS/pbXU=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do=
+github.com/aws/aws-sdk-go-v2/service/signin v1.4.0 h1:sLzmJGCMv+C8KqiJgEqDLB6vxaJGmobRh4rr//ZpA3w=
+github.com/aws/aws-sdk-go-v2/service/signin v1.4.0/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.3/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw=
 github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8=
 github.com/aws/aws-sdk-go-v2/service/sso v1.25.2/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.31.5 h1:xlK3Tdc8FO7Tq1k0+hL+otF33glj+dE+qeM5iINiDvU=
-github.com/aws/aws-sdk-go-v2/service/sso v1.31.5/go.mod h1:u8af9Nqkmqnr96f7v9nHqzZT9XBwbXEkTiqT4ROuJSE=
+github.com/aws/aws-sdk-go-v2/service/sso v1.32.0 h1:qjMmry/cBDee1E/2gyvel0uRYCi3mwRZ2hf6N+GAodo=
+github.com/aws/aws-sdk-go-v2/service/sso v1.32.0/go.mod h1:u8af9Nqkmqnr96f7v9nHqzZT9XBwbXEkTiqT4ROuJSE=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.3/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.8 h1:yX1IbiBfC7SdEgDwIGnRaZyPPDRbQPDOJxl8102PcGk=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.8/go.mod h1:DMPWJBjYs6+3+f/qhBFEFPPlQ6NlhWjai3dJNvipJ84=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.0 h1:fpOlDPI55HdszaxapEGk6HsGosOUaM2YPWJpjMgp8UI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.0/go.mod h1:DMPWJBjYs6+3+f/qhBFEFPPlQ6NlhWjai3dJNvipJ84=
 github.com/aws/aws-sdk-go-v2/service/sts v1.18.4/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s=
 github.com/aws/aws-sdk-go-v2/service/sts v1.28.1/go.mod h1:uQ7YYKZt3adCRrdCBREm1CD3efFLOUNH77MrUCvx5oA=
 github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
-github.com/aws/aws-sdk-go-v2/service/sts v1.43.5 h1:T3ANO8QWDbzQD8f4+UaX+fvJlyGnOFMKLbW+NGBHg04=
-github.com/aws/aws-sdk-go-v2/service/sts v1.43.5/go.mod h1:9gdl4RrflIdpDb2TlXshWgR1F9TeCkvqDx77Vpr4Z/Q=
+github.com/aws/aws-sdk-go-v2/service/sts v1.44.0 h1:bLZ0PolJ8J+HkJHztcXORUpHXBye2U8298lCEMi6ZCU=
+github.com/aws/aws-sdk-go-v2/service/sts v1.44.0/go.mod h1:9gdl4RrflIdpDb2TlXshWgR1F9TeCkvqDx77Vpr4Z/Q=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
@@ -602,6 +604,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
@@ -697,8 +701,8 @@ github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/q
 github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
 github.com/containerd/containerd/api v1.11.1 h1:h8nfoDW9+fNsC/9TwiAHj8B1GzXKtR4eFtkhi/X5RLU=
 github.com/containerd/containerd/api v1.11.1/go.mod h1:CaQFRu+N1MtbgL6JDOJLUB1hCKESU1lD6MuTJhgtdlw=
-github.com/containerd/containerd/v2 v2.3.2 h1:eLven1YxRMkeiKu7IcMrPKE+gn8sGR1DqHbbshMEvWM=
-github.com/containerd/containerd/v2 v2.3.2/go.mod h1:rHKGm3VW6wNrINb3x8mNT+w7qYXFVElTt/8HTuxVhD4=
+github.com/containerd/containerd/v2 v2.3.3 h1:MUNBVVBTBpPll7KPh5GTvkC3cfG03PQLAHVdsUoue9k=
+github.com/containerd/containerd/v2 v2.3.3/go.mod h1:rHKGm3VW6wNrINb3x8mNT+w7qYXFVElTt/8HTuxVhD4=
 github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg=
 github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
 github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
@@ -725,12 +729,14 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/cyphar/filepath-securejoin v0.6.0 h1:BtGB77njd6SVO6VztOHfPxKitJvd/VPT+OFBFMOi1Is=
+github.com/cyphar/filepath-securejoin v0.6.0/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgraph-io/badger/v4 v4.9.2 h1:Wb5qw8gElqwV1a8msHTeQKova9b1V10heFKMIiPd80E=
-github.com/dgraph-io/badger/v4 v4.9.2/go.mod h1:nJjaJTUOSsQEBhsq209FmwCvMJzEA3e74RjZw6V2pQI=
+github.com/dgraph-io/badger/v4 v4.9.4 h1:bcw+waCpzRZ2nmcSPbnPvDVhiEsn98TKmvnAhK7r7LM=
+github.com/dgraph-io/badger/v4 v4.9.4/go.mod h1:nJjaJTUOSsQEBhsq209FmwCvMJzEA3e74RjZw6V2pQI=
 github.com/dgraph-io/ristretto/v2 v2.4.0 h1:I/w09yLjhdcVD2QV192UJcq8dPBaAJb9pOuMyNy0XlU=
 github.com/dgraph-io/ristretto/v2 v2.4.0/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
@@ -813,8 +819,8 @@ github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8
 github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
 github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
-github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
+github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8=
+github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
 github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g=
 github.com/go-fonts/dejavu v0.3.2/go.mod h1:m+TzKY7ZEl09/a17t1593E4VYW8L1VaBXHzFZOIjGEY=
 github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks=
@@ -1103,6 +1109,8 @@ github.com/hamba/avro/v2 v2.17.2/go.mod h1:Q9YK+qxAhtVrNqOhwlZTATLgLA8qxG2vtvkhK
 github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
 github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
@@ -1118,6 +1126,8 @@ github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP
 github.com/hashicorp/go-msgpack/v2 v2.1.5 h1:Ue879bPnutj/hXfmUk6s/jtIK90XxgiUIcXRl656T44=
 github.com/hashicorp/go-msgpack/v2 v2.1.5/go.mod h1:bjCsRXpZ7NsJdk45PoCQnzRGDaK8TKm5ZnDI/9y3J4M=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
@@ -1282,6 +1292,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
+github.com/moby/sys/capability v0.4.0 h1:4D4mI6KlNtWMCM1Z/K0i7RV1FkX+DBDHKVJpCndZoHk=
+github.com/moby/sys/capability v0.4.0/go.mod h1:4g9IK291rVkms3LKCDOoYlnV8xKwoDTpIrNEE35Wq0I=
 github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg=
 github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4=
 github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
@@ -1303,10 +1315,10 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/predastore v1.11.0 h1:FD/Gk7cXhMZNcRBXectwBcR7jsMoS77WM3jcFgNVzY0=
-github.com/mulgadc/predastore v1.11.0/go.mod h1:gPrTLe9jY3CQyNtlpFurzUp2D0e8wPtVw/v48BmkhKA=
-github.com/mulgadc/viperblock v1.11.0 h1:/xIDWhwRIKkWiD1pvXH/f3aQe2SBH6MYQAlStNdek9Y=
-github.com/mulgadc/viperblock v1.11.0/go.mod h1:B4upzru+3opexPHNp6jkIqFvr8rdNs9lVVb+FJUd+9M=
+github.com/mulgadc/predastore v1.12.0 h1:JmoF//cIr3DgJhGypNVVLtaBZGCVUDzOo6z63407x54=
+github.com/mulgadc/predastore v1.12.0/go.mod h1:kgTncuCaB11e3eCQeZF0ydEFG0biuOlrZ4DlHNzwvZY=
+github.com/mulgadc/viperblock v1.12.0 h1:1PYG7GQ1cBbHbLI7QB7IzVu6EeKhrtcAJ+VAvNczjs8=
+github.com/mulgadc/viperblock v1.12.0/go.mod h1:5MGciKEvqpwLv2pdjk9j7PR8NHtXPs4pJzJi4UJojeU=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -1340,6 +1352,10 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw
 github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
 github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg=
 github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116 h1:tAKu3NkKWZYpqBSOJKwTxT1wIGueiF7gcmcNgr5pNTY=
+github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116/go.mod h1:DKDEfzxvRkoQ6n9TGhxQgg2IM1lY4aM0eaQP4e3oElw=
+github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22F+ISDCJE=
+github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg=
 github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -1355,8 +1371,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
-github.com/pelletier/go-toml/v2 v2.4.2 h1:M2fKKbmyvI+hGId/D0W64qDBMVhJnNR10O5gIbMc//Q=
-github.com/pelletier/go-toml/v2 v2.4.2/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
+github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
 github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
@@ -1557,6 +1573,12 @@ github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 h1:tHNk7XK9GkmKUR6Gh8gV
 github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4=
 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@@ -1596,6 +1618,8 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/contrib/bridges/otelslog v0.19.0 h1:5RgvxieNq9tS3ewrV1vnODvbHPfKUIJcYtF9Cvz+6aQ=
+go.opentelemetry.io/contrib/bridges/otelslog v0.19.0/go.mod h1:iTBIdNwx/xmUhfgJs6+84S4dIK059811cO1eUBjKcHY=
 go.opentelemetry.io/contrib/detectors/gcp v1.28.0/go.mod h1:9BIqH22qyHWAiZxQh0whuJygro59z+nbMVuc7ciiGug=
 go.opentelemetry.io/contrib/detectors/gcp v1.29.0/go.mod h1:GW2aWZNwR2ZxDLdv8OyC2G8zkRoQBuURgV7RPQgcPoU=
 go.opentelemetry.io/contrib/detectors/gcp v1.31.0/go.mod h1:tzQL6E1l+iV44YFTkcAeNQqzXUiekSYP9jjJjXwEd00=
@@ -1658,6 +1682,8 @@ go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOX
 go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
 go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
 go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0 h1:rydZ9sxbcFdm/oWrVyfLTjHIygMgv0bEeMd+3B/BvoM=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0/go.mod h1:earQ25dooT0Hhspq59DZ8YCC50jWfOlFEeWoxy/P444=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 h1:SUplec5dp06reu1zaXmOXdvqH398taqrDXqUl99jxSc=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0/go.mod h1:ho2g4N+ane+swq5I/VBkKWnRDY4kUINH3FuqyZqX/Ug=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI=
@@ -1669,6 +1695,8 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zb
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw=
+go.opentelemetry.io/otel/log v0.20.0 h1:/5i0vuHxCLWUfChWG41K9wkM0jafruPw9NU1/RCJirs=
+go.opentelemetry.io/otel/log v0.20.0/go.mod h1:wOcMcjsZpG8x7Bak7IhSi/lg8wscV2C1VdrKCLPlt0E=
 go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
 go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
 go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
@@ -1716,6 +1744,10 @@ go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWm
 go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
 go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
 go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
+go.opentelemetry.io/otel/sdk/log v0.20.0 h1:vM3xI7TQgKPiSghe6urZtAkyFY7SodrSpC83CffDFuY=
+go.opentelemetry.io/otel/sdk/log v0.20.0/go.mod h1:Knej2nmsTUzN79T2eeXdRsjjPcoxoq2pUyUHz9TFyyU=
+go.opentelemetry.io/otel/sdk/log/logtest v0.20.0 h1:OqdRZ1guyzamK3M6LlRsmGqRrjkHWw6WZOKKli5ELpg=
+go.opentelemetry.io/otel/sdk/log/logtest v0.20.0/go.mod h1:PuMIlm7zAt7c3z8zfOI5ox4iT1Z87We+PF6YoINux/M=
 go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg=
 go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ=
 go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y=
@@ -1842,8 +1874,8 @@ golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVo
 golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
 golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
 golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
-golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
-golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
+golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
+golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1955,8 +1987,8 @@ golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
 golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
 golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
 golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
-golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
-golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2162,8 +2194,9 @@ golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
-golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
 golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -2306,8 +2339,8 @@ golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
-golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
 golang.org/x/telemetry v0.0.0-20250710130107-8d8967aff50b/go.mod h1:4ZwOYna0/zsOKwuR5X/m0QFOJpSZvAxFfkQT+Erd9D4=
@@ -2320,8 +2353,8 @@ golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3c
 golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548=
 golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw=
 golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE=
-golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 h1:HjU6IWBiAgRIdAJ9/y1rwCn+UELEmwV+VsTLzj/W4sE=
-golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6/go.mod h1:Eqhaxk/wZsWEH8CRxLwj6xzEJbz7k1EFGqx7nyCoabE=
+golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc=
+golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2369,8 +2402,8 @@ golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
 golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
 golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
 golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
-golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
-golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2415,8 +2448,8 @@ golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
 golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
 golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
-golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
-golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
+golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2531,8 +2564,8 @@ golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg
 golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
 golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
 golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
-golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
-golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
 golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
 golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM=
@@ -3182,4 +3215,10 @@ rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
 sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
+tags.cncf.io/container-device-interface v1.1.0 h1:RnxNhxF1JOu6CJUVpetTYvrXHdxw9j9jFYgZpI+anSY=
+tags.cncf.io/container-device-interface v1.1.0/go.mod h1:76Oj0Yqp9FwTx/pySDc8Bxjpg+VqXfDb50cKAXVJ34Q=
+tags.cncf.io/container-device-interface/specs-go v1.1.0 h1:QRZVeAceQM+zTZe12eyfuJuuzp524EKYwhmvLd+h+yQ=
+tags.cncf.io/container-device-interface/specs-go v1.1.0/go.mod h1:u86hoFWqnh3hWz3esofRFKbI261bUlvUfLKGrDhJkgQ=

internal/gwsign/signer_test.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/mulgadc/predastore/auth"
+	"github.com/mulgadc/predastore/pkg/sigv4"
 )
 
 // failingProvider models a cold IMDS datapath: Retrieve always errors, as the
@@ -40,13 +40,14 @@ func TestSign_StaticVerifies(t *testing.T) {
 
 	var verifyErr error
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		req, err := auth.ParseReq(r)
+		// sigv4.Parse reads and hashes the body itself, so no explicit body hash.
+		sr, err := sigv4.Parse(r)
 		if err != nil {
 			verifyErr = err
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-		verifyErr = req.Verify(testSecretKey, testService, testRegion, auth.WithBodyHash(payloadHash))
+		_, verifyErr = sr.Verify(testSecretKey, testRegion, testService)
 		w.WriteHeader(http.StatusOK)
 	}))
 	defer srv.Close()

scripts/build-system-image.sh

@@ -72,8 +72,10 @@ for field in IMAGE_NAME IMAGE_SIZE; do
     fi
 done
 
-# Derived paths
-BUILD_DIR="/tmp/${IMAGE_NAME}-image-build"
+# Derived paths. BUILD_DIR defaults to /tmp, but Ubuntu GPU images are 16G and a
+# typical /tmp is a RAM-backed tmpfs too small to hold the qcow2 + expanded raw,
+# so allow an override onto a real disk (SYSTEM_IMAGE_BUILD_DIR).
+BUILD_DIR="${SYSTEM_IMAGE_BUILD_DIR:-/tmp}/${IMAGE_NAME}-image-build"
 
 case "$DISTRO" in
     alpine)
@@ -116,7 +118,7 @@ import_ami() {
         args+=(--ami-name "$AMI_NAME")
     fi
     if [[ -n "${SYSTEM_TAG:-}" ]]; then
-        args+=(--tag "$SYSTEM_TAG")
+        for _tag in ${SYSTEM_TAG}; do args+=(--tag "$_tag"); done
     fi
     spx admin images import "${args[@]}"
 }
@@ -367,7 +369,9 @@ else
     fi
     if [[ -n "${SYSTEM_TAG:-}" ]]; then
         echo -e "    --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64 --boot-mode ${BOOT_MODE}${NAME_HINT} \\"
-        echo "    --tag ${SYSTEM_TAG}"
+        _tag_line=""
+        for _tag in ${SYSTEM_TAG}; do _tag_line="${_tag_line} --tag ${_tag}"; done
+        echo "   ${_tag_line}"
     else
         echo -e "    --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64 --boot-mode ${BOOT_MODE}${NAME_HINT}"
     fi

scripts/images/ecs-agent-gpu/ecs-agent.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Spinifex ECS container-instance agent
+# Resolves host identity from IMDSv2, registers + heartbeats + polls task
+# assignments through the AWS gateway over TLS+SigV4 (no NATS), and pulls
+# task images through containerd.
+Requires=containerd.service
+After=containerd.service ecs-ca-install.service network-online.target mulga-cdi-generate.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+# Static config (gateway URL, CA, region, cluster, seeded IAM creds,
+# containerd socket) comes from /etc/spinifex-ecs/agent.env, dropped by
+# cloud-init at launch. Absent file is tolerated — the agent falls back to
+# its built-in defaults.
+EnvironmentFile=-/etc/spinifex-ecs/agent.env
+ExecStart=/usr/local/bin/ecs-agent
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

scripts/images/ecs-agent-gpu/ecs-ca-install.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Install Spinifex gateway CA into the system trust (first boot)
+# The CA is delivered per-instance (not baked) at
+# /etc/spinifex-ecs/gateway-ca.pem by cloud-init user-data, mirroring how EKS
+# delivers EKS_GATEWAY_CA. No-op when the file is absent — containerd then
+# relies on the stock system trust.
+Before=containerd.service ecs-agent.service
+After=local-fs.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/sh -c '\
+    if [ -f /etc/spinifex-ecs/gateway-ca.pem ]; then \
+        install -D -m 0644 /etc/spinifex-ecs/gateway-ca.pem /usr/local/share/ca-certificates/spinifex-gateway.crt && \
+        update-ca-certificates >/dev/null 2>&1; \
+    else \
+        echo "no gateway CA at /etc/spinifex-ecs/gateway-ca.pem; using stock system trust"; \
+    fi'
+
+[Install]
+WantedBy=multi-user.target

scripts/images/ecs-agent-gpu/manifest.conf

@@ -0,0 +1,36 @@
+IMAGE_NAME="ecs-agent-gpu"
+IMAGE_DESCRIPTION="Ubuntu ECS GPU container-instance — containerd + runc + ecs-agent + NVIDIA server driver + nvidia-container-toolkit"
+DISTRO="ubuntu"
+UBUNTU_VERSION="26.04"
+UBUNTU_CODENAME="resolute"
+
+# 16G — same headroom rationale as eks-node-gpu: kernel headers, NVIDIA
+# driver + DKMS build artefacts, nvidia-container-toolkit.
+IMAGE_SIZE="16G"
+
+# Ubuntu's containerd apt package ships its own systemd unit (unlike Alpine's
+# containerd-openrc, which needed one written) — nothing to author for it
+# here, just `systemctl enable containerd` below. nvidia-container-toolkit
+# lives in its own NVIDIA-hosted apt repo, not enabled on the base cloud
+# image, so it is added+installed in setup.sh rather than here (see
+# scripts/images/ubuntu-gpu-nvidia-setup.sh).
+APT_PACKAGES="containerd runc containernetworking-plugins ca-certificates curl iproute2 initramfs-tools"
+
+# ecs-agent is the only built binary (static, FIPS boot like the host build) —
+# BUILD_COMMANDS/INSTALL_BINARIES copied verbatim from scripts/images/ecs-agent/.
+BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags \"-s -w\" -o bin/ecs-agent ./cmd/ecs-agent"
+
+INSTALL_BINARIES="bin/ecs-agent:/usr/local/bin/ecs-agent"
+
+INSTALL_FILES="\
+scripts/images/ecs-agent/10-bridge.conflist:/etc/cni/net.d/10-bridge.conflist \
+scripts/images/ecs-agent-gpu/ecs-agent.service:/etc/systemd/system/ecs-agent.service \
+scripts/images/ecs-agent-gpu/ecs-ca-install.service:/etc/systemd/system/ecs-ca-install.service \
+scripts/images/ecs-agent-gpu/mulga-cdi-generate.service:/etc/systemd/system/mulga-cdi-generate.service"
+
+ENABLE_SERVICES="containerd ecs-ca-install ecs-agent mulga-cdi-generate"
+
+SETUP_SCRIPT="scripts/images/ecs-agent-gpu/setup.sh"
+
+AMI_NAME="spinifex-ecs-node-gpu"
+SYSTEM_TAG="spinifex:managed-by=ecs gpu-vendor=nvidia"

scripts/images/ecs-agent-gpu/mulga-cdi-generate.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Generate NVIDIA CDI spec for the container runtime (GPU-gated, first boot)
+After=systemd-modules-load.service
+Before=containerd.service ecs-agent.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+# Silently skip on non-GPU boots: only generate when an NVIDIA GPU is
+# actually present and the driver has bound.
+ExecCondition=/bin/sh -c 'command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1'
+ExecStartPre=/bin/mkdir -p /etc/cdi
+ExecStart=/usr/bin/nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+
+[Install]
+WantedBy=multi-user.target

scripts/images/ecs-agent-gpu/setup.sh

@@ -0,0 +1,155 @@
+#!/bin/sh
+set -eu
+
+# setup.sh — guest customisation for the ecs-agent-gpu AMI (Ubuntu ECS GPU
+# container-instance).
+#
+# Runs inside the libguestfs appliance (via virt-customize --run) under
+# build-system-image.sh, after packages, binaries and INSTALL_FILES are
+# placed. Wires the CNI bin path containerd's CRI plugin expects (mirroring
+# scripts/images/ecs-agent/setup.sh, but Ubuntu's containernetworking-plugins
+# package lays out differently to Alpine's — see below), preps the containerd
+# content store + agent config dir, and bakes the headless NVIDIA driver +
+# nvidia-container-toolkit (steps reused from
+# scripts/images/ubuntu-gpu-nvidia-setup.sh — see install_nvidia_gpu_stack()
+# below, duplicated verbatim from scripts/images/eks-node-gpu/setup.sh so
+# both GPU node images stay consistent; each image builds standalone from
+# its own manifest so there is no shared-script mechanism to source from).
+#
+# Unlike scripts/images/ecs-agent/setup.sh there is no chmod dance for OpenRC
+# init scripts (systemd unit files don't need an exec bit) and no SYSLINUX/
+# serial-console tweaks (Ubuntu boots via GRUB/UEFI, not Alpine's extlinux).
+
+# Ubuntu's containernetworking-plugins package installs plugin binaries
+# directly under /usr/lib/cni (verified via `dpkg -L containernetworking-plugins`
+# — no /bin subdir, unlike Alpine's /usr/libexec/cni). containerd's CRI plugin
+# defaults its bin dir to /opt/cni/bin. Symlink so the baked bridge conflist
+# resolves its plugins when task networking is wired up.
+mkdir -p /opt/cni
+ln -sf /usr/lib/cni /opt/cni/bin
+
+# Content-store + agent config dirs.
+mkdir -p /var/lib/containerd /etc/spinifex-ecs
+
+# ── NVIDIA headless driver + container toolkit (node-image subset) ──────────
+# Reused from scripts/images/ubuntu-gpu-nvidia-setup.sh: kernel detection,
+# linux-headers-<kver>, the latest versioned nvidia-dkms-<ver>-server +
+# nvidia-utils-<ver>-server, `dkms install`, nvidia.ko verification,
+# `apt-mark hold` on the kernel/nvidia/libnvidia packages, the nouveau
+# modprobe blacklist, and `update-initramfs -u -k <kver>`.
+#
+# Dropped for a node image: pciutils/python3/git/curl-extras/htop/tmux/ffmpeg/
+# libgl1/libglib dev payload, and Docker CE entirely — GPU nodes run
+# containerd (this image) or k3s (scripts/images/eks-node-gpu), never Docker.
+# `nvidia-ctk runtime configure --runtime=docker` and the docker-group
+# cloud-init drop-in are dropped for the same reason; the runtime is instead
+# wired via CDI at first boot by mulga-cdi-generate.service, gated on an
+# NVIDIA GPU actually being present.
+install_nvidia_gpu_stack() {
+    KVER=$(ls /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1 | sed 's|/boot/vmlinuz-||') || true
+    if [ -z "${KVER}" ]; then
+        # Ubuntu 26.04 minimal cloud image keeps the kernel on the ESP (not
+        # mounted in the appliance chroot), so /boot is empty — fall back to
+        # /lib/modules.
+        KVER=$(ls /lib/modules/ 2>/dev/null | sort -V | tail -1)
+    fi
+    if [ -z "${KVER}" ]; then
+        echo "ERROR: No kernel found in /boot or /lib/modules — base image may be missing a kernel"
+        exit 1
+    fi
+    echo "[nvidia-gpu-stack] target kernel: ${KVER}"
+
+    # Prevent kernel postinst hooks from attempting grub-install against the
+    # host disk — this fails silently inside the appliance and can abort the
+    # postinst, leaving /boot incomplete.
+    cat > /etc/kernel-img.conf <<'EOF'
+do_symlinks = yes
+do_bootloader = no
+do_initrd = yes
+link_in_boot = yes
+EOF
+
+    apt-get install -y --no-install-recommends "linux-headers-${KVER}"
+
+    # Ubuntu 26.04+ no longer ships unversioned meta-packages; detect the
+    # latest available versioned NVIDIA server driver.
+    NVIDIA_VER=$(apt-cache search '^nvidia-dkms-[0-9]+-server$' 2>/dev/null \
+        | grep -oP '(?<=nvidia-dkms-)\d+(?=-server)' | sort -rn | head -1)
+    if [ -z "${NVIDIA_VER}" ]; then
+        echo "ERROR: No versioned nvidia-dkms-*-server package found in apt cache"
+        apt-cache search nvidia-dkms || true
+        exit 1
+    fi
+    echo "[nvidia-gpu-stack] installing NVIDIA server driver version: ${NVIDIA_VER}"
+    apt-get install -y --no-install-recommends \
+        "nvidia-dkms-${NVIDIA_VER}-server" \
+        "nvidia-utils-${NVIDIA_VER}-server"
+
+    NVIDIA_DKMS_NAME=$(dkms status 2>/dev/null | awk -F'[,/ ]+' '/nvidia/{print $1; exit}')
+    NVIDIA_DKMS_VER=$(dkms status 2>/dev/null  | awk -F'[,/ ]+' '/nvidia/{print $2; exit}')
+    if [ -n "${NVIDIA_DKMS_NAME}" ] && [ -n "${NVIDIA_DKMS_VER}" ]; then
+        echo "[nvidia-gpu-stack] building NVIDIA DKMS module ${NVIDIA_DKMS_NAME}/${NVIDIA_DKMS_VER} for kernel ${KVER}..."
+        dkms install -m "${NVIDIA_DKMS_NAME}" -v "${NVIDIA_DKMS_VER}" --kernelver "${KVER}" --force
+    else
+        echo "WARNING: nvidia DKMS module not found in dkms status — driver install may have failed"
+    fi
+
+    NVIDIA_KO=$(find "/lib/modules/${KVER}/updates" -name "nvidia.ko*" 2>/dev/null | head -1)
+    if [ -z "${NVIDIA_KO}" ]; then
+        echo "ERROR: nvidia.ko not found under /lib/modules/${KVER}/updates after DKMS build"
+        exit 1
+    fi
+    echo "[nvidia-gpu-stack] NVIDIA kernel module: ${NVIDIA_KO}"
+
+    # Pin the exact kernel + headers + nvidia/libnvidia packages so
+    # unattended-upgrades cannot replace what the DKMS module was built for.
+    apt-mark hold "linux-image-${KVER}" "linux-headers-${KVER}"
+
+    NVIDIA_PKGS=$(dpkg --get-selections 'nvidia-*' 2>/dev/null | awk '/install$/{print $1}' || true)
+    LIBNVIDIA_PKGS=$(dpkg --get-selections 'libnvidia-*' 2>/dev/null | awk '/install$/{print $1}' || true)
+    # shellcheck disable=SC2086
+    [ -n "${NVIDIA_PKGS}" ]    && apt-mark hold ${NVIDIA_PKGS}
+    # shellcheck disable=SC2086
+    [ -n "${LIBNVIDIA_PKGS}" ] && apt-mark hold ${LIBNVIDIA_PKGS}
+
+    # nvidia.ko and nouveau conflict; without this the NVIDIA driver will not
+    # bind after boot.
+    cat > /etc/modprobe.d/blacklist-nouveau.conf <<'EOF'
+blacklist nouveau
+options nouveau modeset=0
+EOF
+
+    # Load the NVIDIA modules at boot so /dev/nvidia* exist before CDI
+    # generation and the container runtime start — a headless node never runs
+    # an X server or CUDA app to trigger the on-demand load.
+    cat > /etc/modules-load.d/nvidia.conf <<'EOF'
+nvidia
+nvidia_uvm
+nvidia_modeset
+EOF
+
+    # nvidia-container-toolkit lives in its own NVIDIA-hosted apt repo, which
+    # is not enabled on the base Ubuntu cloud image — add it here (same
+    # gpg key + list as scripts/images/ubuntu-gpu-nvidia-setup.sh) and
+    # install only the headless toolkit (no Docker CE).
+    apt-get install -y --no-install-recommends gnupg
+    mkdir -p /usr/share/keyrings /etc/apt/sources.list.d
+    curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
+        | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
+    curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
+        | sed 's|deb https://|deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://|g' \
+        > /etc/apt/sources.list.d/nvidia-container-toolkit.list
+    apt-get update -qq
+    apt-get install -y --no-install-recommends nvidia-container-toolkit
+    apt-mark hold nvidia-container-toolkit nvidia-container-toolkit-base \
+        libnvidia-container1 libnvidia-container-tools 2>/dev/null || true
+
+    # Rebuild initramfs with nouveau blacklisted and the NVIDIA module included.
+    update-initramfs -u -k "${KVER}"
+
+    echo "[nvidia-gpu-stack] done: kernel=${KVER}, NVIDIA headless driver + nvidia-container-toolkit installed"
+}
+
+install_nvidia_gpu_stack
+
+echo "[ecs-agent-gpu-setup] done"

scripts/images/eks-node-gpu/99-mulga-ebs-byid.rules

@@ -0,0 +1,14 @@
+# Mint the EBS-style /dev/disk/by-id symlink the upstream aws-ebs-csi-driver
+# node plugin resolves, for hot-attached Viperblock virtio-blk volumes (vd*,
+# serial = volume-id). This is the udev equivalent of the Alpine mdev rule
+# wired by scripts/images/eks-node/setup.sh into /etc/mdev.conf.
+#
+# The shared helper /usr/local/sbin/mulga-ebs-byid (reused verbatim from
+# scripts/images/eks-node/mulga-ebs-byid.sh) reads the device name from
+# $MDEV or, if unset, $1 — so passing the kernel name via %k as the sole
+# argument works without any udev-specific shim. udev already sets ACTION to
+# add/remove, matching the values the helper's case statement expects.
+# Already-attached devices at boot are coldplugged separately by
+# mulga-ebs-byid.service; this rule only covers hotplug events.
+ACTION=="add", SUBSYSTEM=="block", KERNEL=="vd[a-z]", RUN+="/usr/local/sbin/mulga-ebs-byid %k"
+ACTION=="remove", SUBSYSTEM=="block", KERNEL=="vd[a-z]", RUN+="/usr/local/sbin/mulga-ebs-byid %k"

scripts/images/eks-node-gpu/k3s-agent-prestart.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+set -eu
+
+# k3s-agent-prestart — reproduces the start_pre logic from
+# scripts/images/eks-node/k3s-agent.initd for the systemd unit: requires
+# K3S_URL + K3S_TOKEN from /etc/spinifex-eks/agent.env (sourced by the unit's
+# own EnvironmentFile=), then resolves the kubelet provider-id from IMDSv2
+# (aws:///<az>/<instance-id>, EKS parity so the AWS Load Balancer Controller
+# can resolve nodes for ip-mode target registration) and writes it as an
+# extra k3s agent argument for the unit's ExecStart to source.
+#
+# Run as ExecStartPre=; a nonzero exit here fails k3s-agent.service before k3s
+# is ever invoked, same as the OpenRC start_pre returning 1.
+
+ENV_FILE=/etc/spinifex-eks/agent.env
+DROPIN=/run/k3s-agent-extra-args.env
+LOGTAG="k3s-agent-prestart"
+
+# EnvironmentFile= already injects agent.env into this process' environment,
+# but source it directly too so this script also works if invoked standalone.
+if [ -f "${ENV_FILE}" ]; then
+    # shellcheck disable=SC1090
+    . "${ENV_FILE}"
+fi
+
+if [ -z "${K3S_URL:-}" ] || [ -z "${K3S_TOKEN:-}" ]; then
+    echo "[${LOGTAG}] K3S_URL and K3S_TOKEN must be set in ${ENV_FILE}" >&2
+    exit 1
+fi
+
+IMDS=http://169.254.169.254/latest
+CURL="curl -fsS --connect-timeout 3 --max-time 5"
+
+EXTRA_ARGS=""
+TOK=$(${CURL} -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 120' "${IMDS}/api/token" 2>/dev/null || true)
+if [ -n "${TOK}" ]; then
+    IID=$(${CURL} -H "X-aws-ec2-metadata-token: ${TOK}" "${IMDS}/meta-data/instance-id" 2>/dev/null || true)
+    AZ=$(${CURL} -H "X-aws-ec2-metadata-token: ${TOK}" "${IMDS}/meta-data/placement/availability-zone" 2>/dev/null || true)
+    if [ -n "${IID}" ] && [ -n "${AZ}" ]; then
+        EXTRA_ARGS="--kubelet-arg=provider-id=aws:///${AZ}/${IID}"
+    else
+        echo "[${LOGTAG}] IMDS instance-id/az empty; keeping default providerID" >&2
+    fi
+else
+    echo "[${LOGTAG}] IMDS token request failed; keeping default providerID" >&2
+fi
+
+echo "K3S_AGENT_EXTRA_ARGS=${EXTRA_ARGS}" > "${DROPIN}"

scripts/images/eks-node-gpu/k3s-agent.service

@@ -0,0 +1,40 @@
+[Unit]
+Description=K3s Kubernetes agent (Mulga EKS GPU worker)
+# K3S_URL points at the cluster's NLB endpoint (https://{cluster}.{account}
+# .eks.{region}.{suffix}:443) so the agent join path stays stable across the
+# v2 HA migration (single-VM v1 -> 3-node etcd quorum v2). No AMI rebuild
+# needed when the control plane expands.
+After=network-online.target mulga-eks-provider-id.service mulga-vpc-mtu.service mulga-ebs-byid.service mulga-cdi-generate.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+# k3s embeds containerd + kubelet in-process, so on a cgroup-v2 host the unit
+# must delegate a cgroup subtree or the kubelet cannot create /kubepods and
+# ContainerManager.Start fails before Node registration (OpenRC ran k3s in the
+# root cgroup, so this was implicit). overlay/br_netfilter feed the embedded
+# containerd snapshotter + kube-proxy.
+Delegate=yes
+KillMode=process
+TasksMax=infinity
+LimitNOFILE=1048576
+LimitNPROC=infinity
+LimitCORE=infinity
+# Cloud-init drops K3S_URL/K3S_TOKEN/K3S_NODE_NAME/K3S_NODE_LABEL here at
+# RunInstances time per the Nodegroup bootstrap flow.
+EnvironmentFile=-/etc/spinifex-eks/agent.env
+# Tee k3s (and its embedded kubelet's klog) stderr to the serial console as well
+# as journald: Ubuntu logs to journald only, which is off-box-invisible on a
+# worker in an isolated customer VPC, so a kubelet that fails to register the
+# Node leaves no trace. The serial console is captured per-VM by the daemon.
+StandardOutput=journal+console
+StandardError=journal+console
+ExecStartPre=-/sbin/modprobe overlay
+ExecStartPre=-/sbin/modprobe br_netfilter
+ExecStartPre=/usr/local/sbin/k3s-agent-prestart
+ExecStart=/bin/sh -c '. /run/k3s-agent-extra-args.env 2>/dev/null || true; exec /usr/local/bin/k3s agent ${K3S_AGENT_EXTRA_ARGS:-}'
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

scripts/images/eks-node-gpu/manifest.conf

@@ -0,0 +1,52 @@
+IMAGE_NAME="eks-node-gpu"
+IMAGE_DESCRIPTION="Ubuntu EKS K3s GPU worker node — NVIDIA server driver (DKMS pre-built) + nvidia-container-toolkit; agent-only (GPU nodes never run the EKS control plane)"
+DISTRO="ubuntu"
+UBUNTU_VERSION="26.04"
+UBUNTU_CODENAME="resolute"
+
+# 16G — headroom for kernel headers, NVIDIA driver + DKMS build artefacts, and
+# nvidia-container-toolkit, mirroring scripts/images/ubuntu-gpu-nvidia.conf.
+IMAGE_SIZE="16G"
+
+# K3s bundles its own containerd, so no containerd/runc package here. The
+# NVIDIA server driver + nvidia-utils come from the stock Ubuntu repos and are
+# installed in setup.sh (kernel-version-pinned DKMS build, like the ubuntu-gpu
+# recipe). nvidia-container-toolkit lives in its own NVIDIA-hosted apt repo,
+# which is not enabled on the base cloud image, so it is also added+installed
+# in setup.sh rather than here (see scripts/images/ubuntu-gpu-nvidia-setup.sh).
+APT_PACKAGES="ca-certificates curl jq iptables conntrack cgroup-tools initramfs-tools"
+
+# ecr-credential-provider is the ONLY server binary a GPU worker needs: the
+# nodegroup userdata points kubelet at it (image-credential-provider-config +
+# bin-dir=/usr/local/bin) for ECR pulls, and kubelet treats a referenced-but-
+# missing provider binary as a FATAL init error (RegisterCredentialProviderPlugins),
+# so the node never registers. The Alpine eks-node ships it; the GPU AMI must too.
+# Server-only binaries (token-webhook, gateway-*, konnectivity, k3s server) stay
+# omitted — GPU nodes are agent-only.
+BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/ecr-credential-provider ./cmd/ecr-credential-provider"
+INSTALL_BINARIES="bin/ecr-credential-provider:/usr/local/bin/ecr-credential-provider"
+
+# Role-agnostic helper scripts are reused verbatim from scripts/images/eks-node/
+# (only the init wiring changes: OpenRC -> systemd). k3s-agent-prestart is new —
+# it reproduces the k3s-agent.initd start_pre logic (env require + IMDS
+# provider-id) for the systemd unit. Server-only units (role selector, k3s
+# server, eks-token-webhook, konnectivity, state-report, k3s recovery,
+# mgmt-net) are intentionally not ported — GPU nodes are single-NIC workers.
+INSTALL_FILES="\
+scripts/images/eks-node/mulga-ebs-byid.sh:/usr/local/sbin/mulga-ebs-byid \
+scripts/images/eks-node/mulga-eks-provider-id.sh:/usr/local/sbin/mulga-eks-provider-id \
+scripts/images/eks-node/mulga-vpc-mtu.sh:/usr/local/sbin/mulga-vpc-mtu \
+scripts/images/eks-node-gpu/k3s-agent-prestart.sh:/usr/local/sbin/k3s-agent-prestart \
+scripts/images/eks-node-gpu/k3s-agent.service:/etc/systemd/system/k3s-agent.service \
+scripts/images/eks-node-gpu/mulga-ebs-byid.service:/etc/systemd/system/mulga-ebs-byid.service \
+scripts/images/eks-node-gpu/mulga-eks-provider-id.service:/etc/systemd/system/mulga-eks-provider-id.service \
+scripts/images/eks-node-gpu/mulga-vpc-mtu.service:/etc/systemd/system/mulga-vpc-mtu.service \
+scripts/images/eks-node-gpu/mulga-cdi-generate.service:/etc/systemd/system/mulga-cdi-generate.service \
+scripts/images/eks-node-gpu/99-mulga-ebs-byid.rules:/etc/udev/rules.d/99-mulga-ebs-byid.rules"
+
+ENABLE_SERVICES="k3s-agent mulga-ebs-byid mulga-eks-provider-id mulga-vpc-mtu mulga-cdi-generate"
+
+SETUP_SCRIPT="scripts/images/eks-node-gpu/setup.sh"
+
+AMI_NAME="spinifex-eks-node-gpu"
+SYSTEM_TAG="spinifex:managed-by=eks gpu-vendor=nvidia"

scripts/images/eks-node-gpu/mulga-cdi-generate.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Generate NVIDIA CDI spec for the container runtime (GPU-gated, first boot)
+After=systemd-modules-load.service
+Before=k3s-agent.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+# Silently skip on non-GPU boots: only generate when an NVIDIA GPU is
+# actually present and the driver has bound.
+ExecCondition=/bin/sh -c 'command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1'
+ExecStartPre=/bin/mkdir -p /etc/cdi
+ExecStart=/usr/bin/nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+
+[Install]
+WantedBy=multi-user.target

scripts/images/eks-node-gpu/mulga-ebs-byid.service

@@ -0,0 +1,18 @@
+[Unit]
+# Role-agnostic boot oneshot that coldplugs the EBS by-id symlink bridge for
+# the aws-ebs-csi-driver node plugin — any virtio-blk volume already attached
+# at boot. Hot-attached volumes are handled separately by the udev rule
+# 99-mulga-ebs-byid.rules, which fires the same helper (see
+# scripts/images/eks-node-gpu/99-mulga-ebs-byid.rules).
+Description=Mulga EBS /dev/disk/by-id symlink bridge (coldplug)
+After=local-fs.target
+Before=k3s-agent.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+Environment=ACTION=coldplug
+ExecStart=/usr/local/sbin/mulga-ebs-byid
+
+[Install]
+WantedBy=multi-user.target

scripts/images/eks-node-gpu/mulga-eks-provider-id.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Set K3s provider-id + topology labels from IMDS for EBS CSI
+After=network-online.target
+Wants=network-online.target
+Before=k3s-agent.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/local/sbin/mulga-eks-provider-id
+
+[Install]
+WantedBy=multi-user.target

scripts/images/eks-node-gpu/mulga-vpc-mtu.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Pin VPC data NIC MTU so flannel fits the OVN geneve underlay
+After=network-online.target
+Wants=network-online.target
+Before=k3s-agent.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/local/sbin/mulga-vpc-mtu
+
+[Install]
+WantedBy=multi-user.target

scripts/images/eks-node-gpu/setup.sh

@@ -0,0 +1,174 @@
+#!/bin/sh
+set -eu
+
+# setup.sh — guest customisation for the eks-node-gpu AMI (Ubuntu, K3s agent
+# worker only — GPU nodes never run the EKS control plane, so there is no
+# server/role-selector path here, unlike scripts/images/eks-node/setup.sh).
+#
+# Runs inside the libguestfs appliance (via virt-customize --run) under
+# build-system-image.sh, after packages, binaries and INSTALL_FILES are
+# placed. Downloads + verifies the pinned K3s binary (same recipe as
+# scripts/images/eks-node/setup.sh), sets exec bits on the uploaded shell
+# helpers, and bakes the headless NVIDIA driver + nvidia-container-toolkit
+# (steps reused from scripts/images/ubuntu-gpu-nvidia-setup.sh — see
+# install_nvidia_gpu_stack() below, duplicated verbatim in
+# scripts/images/ecs-agent-gpu/setup.sh so both images stay consistent).
+#
+# Network access inside the appliance is provided by virt-customize --network.
+
+K3S_VERSION="v1.32.5+k3s1"
+K3S_URL_BASE="https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}"
+
+echo "[eks-node-gpu-setup] fetching k3s checksums ${K3S_VERSION}"
+curl -fsSL -o /tmp/k3s-checksums.txt "${K3S_URL_BASE}/sha256sum-amd64.txt"
+K3S_SHA256=$(awk '/[ \t]k3s$/{print $1; exit}' /tmp/k3s-checksums.txt)
+if [ -z "${K3S_SHA256}" ]; then
+    echo "[eks-node-gpu-setup] could not parse k3s sha256 from upstream checksums"
+    cat /tmp/k3s-checksums.txt
+    exit 1
+fi
+echo "[eks-node-gpu-setup] downloading k3s ${K3S_VERSION} (sha256=${K3S_SHA256})"
+curl -fsSL -o /usr/local/bin/k3s "${K3S_URL_BASE}/k3s"
+echo "${K3S_SHA256}  /usr/local/bin/k3s" > /tmp/k3s.sha256
+if ! sha256sum -c /tmp/k3s.sha256; then
+    echo "[eks-node-gpu-setup] k3s SHA256 mismatch — refusing to bake AMI"
+    exit 1
+fi
+chmod 0755 /usr/local/bin/k3s
+ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl
+ln -sf /usr/local/bin/k3s /usr/local/bin/crictl
+ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
+
+# Reused shell helpers + the new prestart wrapper land 0644 via INSTALL_FILES;
+# make them executable. The systemd unit files and the udev rules file do not
+# need an exec bit (unlike OpenRC's /etc/init.d/*).
+chmod 0755 /usr/local/sbin/mulga-ebs-byid /usr/local/sbin/mulga-eks-provider-id \
+    /usr/local/sbin/mulga-vpc-mtu /usr/local/sbin/k3s-agent-prestart
+
+# K3s config dir (mulga-eks-provider-id's drop-in + cloud-init/first-boot
+# agent config) and the shared spinifex-eks state dir (agent.env lands here
+# from cloud-init at launch).
+mkdir -p /etc/rancher/k3s/config.yaml.d /etc/spinifex-eks
+
+# ── NVIDIA headless driver + container toolkit (node-image subset) ──────────
+# Reused from scripts/images/ubuntu-gpu-nvidia-setup.sh: kernel detection,
+# linux-headers-<kver>, the latest versioned nvidia-dkms-<ver>-server +
+# nvidia-utils-<ver>-server, `dkms install`, nvidia.ko verification,
+# `apt-mark hold` on the kernel/nvidia/libnvidia packages, the nouveau
+# modprobe blacklist, and `update-initramfs -u -k <kver>`.
+#
+# Dropped for a node image: pciutils/python3/git/curl-extras/htop/tmux/ffmpeg/
+# libgl1/libglib dev payload, and Docker CE entirely — GPU nodes run k3s
+# (this image) or containerd (scripts/images/ecs-agent-gpu), never Docker.
+# `nvidia-ctk runtime configure --runtime=docker` and the docker-group
+# cloud-init drop-in are dropped for the same reason; the runtime is instead
+# wired via CDI at first boot by mulga-cdi-generate.service, gated on an
+# NVIDIA GPU actually being present.
+install_nvidia_gpu_stack() {
+    KVER=$(ls /boot/vmlinuz-* 2>/dev/null | sort -V | tail -1 | sed 's|/boot/vmlinuz-||') || true
+    if [ -z "${KVER}" ]; then
+        # Ubuntu 26.04 minimal cloud image keeps the kernel on the ESP (not
+        # mounted in the appliance chroot), so /boot is empty — fall back to
+        # /lib/modules.
+        KVER=$(ls /lib/modules/ 2>/dev/null | sort -V | tail -1)
+    fi
+    if [ -z "${KVER}" ]; then
+        echo "ERROR: No kernel found in /boot or /lib/modules — base image may be missing a kernel"
+        exit 1
+    fi
+    echo "[nvidia-gpu-stack] target kernel: ${KVER}"
+
+    # Prevent kernel postinst hooks from attempting grub-install against the
+    # host disk — this fails silently inside the appliance and can abort the
+    # postinst, leaving /boot incomplete.
+    cat > /etc/kernel-img.conf <<'EOF'
+do_symlinks = yes
+do_bootloader = no
+do_initrd = yes
+link_in_boot = yes
+EOF
+
+    apt-get install -y --no-install-recommends "linux-headers-${KVER}"
+
+    # Ubuntu 26.04+ no longer ships unversioned meta-packages; detect the
+    # latest available versioned NVIDIA server driver.
+    NVIDIA_VER=$(apt-cache search '^nvidia-dkms-[0-9]+-server$' 2>/dev/null \
+        | grep -oP '(?<=nvidia-dkms-)\d+(?=-server)' | sort -rn | head -1)
+    if [ -z "${NVIDIA_VER}" ]; then
+        echo "ERROR: No versioned nvidia-dkms-*-server package found in apt cache"
+        apt-cache search nvidia-dkms || true
+        exit 1
+    fi
+    echo "[nvidia-gpu-stack] installing NVIDIA server driver version: ${NVIDIA_VER}"
+    apt-get install -y --no-install-recommends \
+        "nvidia-dkms-${NVIDIA_VER}-server" \
+        "nvidia-utils-${NVIDIA_VER}-server"
+
+    NVIDIA_DKMS_NAME=$(dkms status 2>/dev/null | awk -F'[,/ ]+' '/nvidia/{print $1; exit}')
+    NVIDIA_DKMS_VER=$(dkms status 2>/dev/null  | awk -F'[,/ ]+' '/nvidia/{print $2; exit}')
+    if [ -n "${NVIDIA_DKMS_NAME}" ] && [ -n "${NVIDIA_DKMS_VER}" ]; then
+        echo "[nvidia-gpu-stack] building NVIDIA DKMS module ${NVIDIA_DKMS_NAME}/${NVIDIA_DKMS_VER} for kernel ${KVER}..."
+        dkms install -m "${NVIDIA_DKMS_NAME}" -v "${NVIDIA_DKMS_VER}" --kernelver "${KVER}" --force
+    else
+        echo "WARNING: nvidia DKMS module not found in dkms status — driver install may have failed"
+    fi
+
+    NVIDIA_KO=$(find "/lib/modules/${KVER}/updates" -name "nvidia.ko*" 2>/dev/null | head -1)
+    if [ -z "${NVIDIA_KO}" ]; then
+        echo "ERROR: nvidia.ko not found under /lib/modules/${KVER}/updates after DKMS build"
+        exit 1
+    fi
+    echo "[nvidia-gpu-stack] NVIDIA kernel module: ${NVIDIA_KO}"
+
+    # Pin the exact kernel + headers + nvidia/libnvidia packages so
+    # unattended-upgrades cannot replace what the DKMS module was built for.
+    apt-mark hold "linux-image-${KVER}" "linux-headers-${KVER}"
+
+    NVIDIA_PKGS=$(dpkg --get-selections 'nvidia-*' 2>/dev/null | awk '/install$/{print $1}' || true)
+    LIBNVIDIA_PKGS=$(dpkg --get-selections 'libnvidia-*' 2>/dev/null | awk '/install$/{print $1}' || true)
+    # shellcheck disable=SC2086
+    [ -n "${NVIDIA_PKGS}" ]    && apt-mark hold ${NVIDIA_PKGS}
+    # shellcheck disable=SC2086
+    [ -n "${LIBNVIDIA_PKGS}" ] && apt-mark hold ${LIBNVIDIA_PKGS}
+
+    # nvidia.ko and nouveau conflict; without this the NVIDIA driver will not
+    # bind after boot.
+    cat > /etc/modprobe.d/blacklist-nouveau.conf <<'EOF'
+blacklist nouveau
+options nouveau modeset=0
+EOF
+
+    # Load the NVIDIA modules at boot so /dev/nvidia* exist before CDI
+    # generation and the container runtime start — a headless node never runs
+    # an X server or CUDA app to trigger the on-demand load.
+    cat > /etc/modules-load.d/nvidia.conf <<'EOF'
+nvidia
+nvidia_uvm
+nvidia_modeset
+EOF
+
+    # nvidia-container-toolkit lives in its own NVIDIA-hosted apt repo, which
+    # is not enabled on the base Ubuntu cloud image — add it here (same
+    # gpg key + list as scripts/images/ubuntu-gpu-nvidia-setup.sh) and
+    # install only the headless toolkit (no Docker CE).
+    apt-get install -y --no-install-recommends gnupg
+    mkdir -p /usr/share/keyrings /etc/apt/sources.list.d
+    curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
+        | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
+    curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
+        | sed 's|deb https://|deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://|g' \
+        > /etc/apt/sources.list.d/nvidia-container-toolkit.list
+    apt-get update -qq
+    apt-get install -y --no-install-recommends nvidia-container-toolkit
+    apt-mark hold nvidia-container-toolkit nvidia-container-toolkit-base \
+        libnvidia-container1 libnvidia-container-tools 2>/dev/null || true
+
+    # Rebuild initramfs with nouveau blacklisted and the NVIDIA module included.
+    update-initramfs -u -k "${KVER}"
+
+    echo "[nvidia-gpu-stack] done: kernel=${KVER}, NVIDIA headless driver + nvidia-container-toolkit installed"
+}
+
+install_nvidia_gpu_stack
+
+echo "[eks-node-gpu-setup] done"

scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/20-node.yaml

@@ -46,6 +46,13 @@ spec:
                 - a1.4xlarge
       nodeSelector:
         kubernetes.io/os: linux
+      # The node plugin must run on every worker, including GPU-only workers
+      # tainted nvidia.com/gpu=present:NoSchedule, or GPU pods there cannot
+      # mount EBS volumes (no node plugin = no CSI NodeStage/NodePublish).
+      tolerations:
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
       serviceAccountName: ebs-csi-node-sa
       terminationGracePeriodSeconds: 30
       priorityClassName: system-node-critical

scripts/images/eks-node/addons/nvidia-device-plugin/0.17.4/10-daemonset.yaml

@@ -0,0 +1,74 @@
+# nvidia-device-plugin 0.17.4 — CDI-mode DaemonSet exposing nvidia.com/gpu
+# allocatable on GPU-tainted nodes. Gated to GPU nodes via nodeSelector on the
+# K3S_NODE_LABEL nvidia.com/gpu.present=true (nodegroup_userdata.go); inert
+# elsewhere. --device-list-strategy=cdi-cri makes the plugin generate a CDI spec
+# for GPU workloads (containerd 2.x resolves the CRI-passed CDI devices; no
+# enable_cdi drop-in needed on k3s v1.32.5). The plugin's OWN container needs
+# NVML + the driver tree to run nvml.Init and generate that spec, so it runs
+# under the k3s-auto-configured "nvidia" runtimeClass with the host root mounted
+# at /driver-root and the host /var/run/cdi mounted writable so the generated
+# spec is visible to containerd. GPU workloads themselves need only
+# resources.limits nvidia.com/gpu — no runtimeClass.
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: nvidia-device-plugin-daemonset
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: nvidia-device-plugin
+spec:
+  selector:
+    matchLabels:
+      name: nvidia-device-plugin-ds
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        name: nvidia-device-plugin-ds
+    spec:
+      nodeSelector:
+        nvidia.com/gpu.present: "true"
+      tolerations:
+      - key: nvidia.com/gpu
+        operator: Exists
+        effect: NoSchedule
+      priorityClassName: "system-node-critical"
+      runtimeClassName: nvidia
+      containers:
+      - image: nvcr.io/nvidia/k8s-device-plugin:v0.17.4
+        name: nvidia-device-plugin-ctr
+        args:
+        - --device-list-strategy=cdi-cri
+        env:
+          - name: FAIL_ON_INIT_ERROR
+            value: "false"
+          - name: NVIDIA_VISIBLE_DEVICES
+            value: all
+          - name: NVIDIA_DRIVER_CAPABILITIES
+            value: all
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: ["ALL"]
+        volumeMounts:
+        - name: kubelet-device-plugins-dir
+          mountPath: /var/lib/kubelet/device-plugins
+        - name: driver-root
+          mountPath: /driver-root
+          readOnly: true
+        - name: cdi-root
+          mountPath: /var/run/cdi
+      volumes:
+      - name: kubelet-device-plugins-dir
+        hostPath:
+          path: /var/lib/kubelet/device-plugins
+          type: Directory
+      - name: driver-root
+        hostPath:
+          path: /
+      - name: cdi-root
+        hostPath:
+          path: /var/run/cdi
+          type: DirectoryOrCreate

scripts/images/eks-node/coredns/coredns-mulga.yaml

@@ -135,6 +135,13 @@ spec:
                     operator: DoesNotExist
       nodeSelector:
         kubernetes.io/os: linux
+      # With internalTrafficPolicy=Local each worker resolves through its OWN
+      # CoreDNS pod, so a GPU-only worker (tainted nvidia.com/gpu=present:
+      # NoSchedule) with no local pod would have no DNS at all for its pods.
+      tolerations:
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
       containers:
         - name: coredns
           image: rancher/mirrored-coredns-coredns:1.12.1

scripts/images/eks-node/eks-node-role.sh

@@ -48,14 +48,26 @@ if [ -z "${ROLE}" ] && [ -f "${AGENT_ENVFILE}" ]; then
     ROLE="agent"
 fi
 
+case "${ROLE}" in
+    server | server-join | agent) ;;
+    *)
+        die "SPINIFEX_K3S_ROLE missing or invalid: '${ROLE}'"
+        ;;
+esac
+
+# Written before the rc-service calls below: mulga-eks-k3s-recovery, started
+# inline for server/server-join, reads ROLE_FILE at the top of its own logic
+# and must find it already in place, not empty.
+mkdir -p "$(dirname "${ROLE_FILE}")"
+printf '%s\n' "${ROLE}" > "${ROLE_FILE}"
+
 case "${ROLE}" in
     server)
         log "configuring server role"
         rc-update add eks-token-webhook default
-        # Pre-k3s etcd recovery: added to the runlevel so it runs before k3s on
-        # every subsequent boot, but not started now — a fresh cluster has no
-        # directive and the role file it guards on is written at the end of this
-        # script. It first acts on the reboot the reconciler triggers.
+        # Pre-k3s etcd recovery: enabled for later boots and started inline
+        # now too, so a restore-snapshot DR seed's directive applies before
+        # k3s starts on this first boot. A plain create has no directive, so it's a no-op.
         rc-update add mulga-eks-k3s-recovery default
         rc-update add k3s default
         # konnectivity-server fronts apiserver egress; every server replica runs
@@ -69,6 +81,7 @@ case "${ROLE}" in
         # (mulga-siv-231.7); server-join nodes do not run it.
         rc-update add mulga-eks-addon-sync default
         rc-service eks-token-webhook start
+        rc-service mulga-eks-k3s-recovery start
         rc-service k3s start
         rc-service konnectivity-server start
         rc-service k3s-first-boot start
@@ -79,12 +92,14 @@ case "${ROLE}" in
         log "configuring server-join role"
         rc-update add eks-token-webhook default
         # Pre-k3s etcd recovery (see server role): server-join members are the
-        # wipe-rejoin followers of a cluster-reset, so they need it too.
+        # wipe-rejoin followers of a cluster-reset, so they need it too, and for
+        # the same first-boot reason must be started here, not deferred.
         rc-update add mulga-eks-k3s-recovery default
         rc-update add k3s default
         rc-update add konnectivity-server default
         rc-update add mulga-eks-state-report default
         rc-service eks-token-webhook start
+        rc-service mulga-eks-k3s-recovery start
         rc-service k3s start
         rc-service konnectivity-server start
         rc-service mulga-eks-state-report start
@@ -99,7 +114,5 @@ case "${ROLE}" in
         ;;
 esac
 
-mkdir -p "$(dirname "${ROLE_FILE}")"
-printf '%s\n' "${ROLE}" > "${ROLE_FILE}"
 rc-update del eks-node-role default 2>/dev/null || true
 log "role '${ROLE}' configured"

scripts/images/eks-node/eks-node-role_test.sh

@@ -65,8 +65,11 @@ check "server: enables k3s-first-boot" "yes" "$(grep -q 'rc-update add k3s-first
 check "server: starts k3s" "yes" "$(grep -q 'rc-service k3s start' "${CALLS}" && echo yes || echo no)"
 check "server: enables konnectivity-server" "yes" "$(grep -q 'rc-update add konnectivity-server default' "${CALLS}" && echo yes || echo no)"
 check "server: enables k3s-recovery" "yes" "$(grep -q 'rc-update add mulga-eks-k3s-recovery default' "${CALLS}" && echo yes || echo no)"
-# Recovery is added to the runlevel (pre-k3s on later boots) but not started now.
-check "server: does not start k3s-recovery" "no" "$(grep -q 'rc-service mulga-eks-k3s-recovery' "${CALLS}" && echo yes || echo no)"
+# Recovery is added to the runlevel (so it also runs pre-k3s on later boots)
+# AND started inline now — a restore-snapshot DR seed's directive must apply
+# on this very first boot, before k3s starts.
+check "server: starts k3s-recovery" "yes" "$(grep -q 'rc-service mulga-eks-k3s-recovery start' "${CALLS}" && echo yes || echo no)"
+check "server: starts k3s-recovery before k3s" "yes" "$(awk '/rc-service mulga-eks-k3s-recovery start/{r=NR} /rc-service k3s start/{k=NR} END{print (r>0 && k>0 && r<k) ? "yes" : "no"}' "${CALLS}")"
 check "server: no k3s-agent" "no" "$(grep -q 'k3s-agent' "${CALLS}" && echo yes || echo no)"
 check "server: self-disables" "yes" "$(grep -q 'rc-update del eks-node-role default' "${CALLS}" && echo yes || echo no)"
 
@@ -81,6 +84,7 @@ check "server-join: enables k3s" "yes" "$(grep -q 'rc-update add k3s default' "$
 check "server-join: enables state-report" "yes" "$(grep -q 'rc-update add mulga-eks-state-report default' "${CALLS}" && echo yes || echo no)"
 check "server-join: enables konnectivity-server" "yes" "$(grep -q 'rc-update add konnectivity-server default' "${CALLS}" && echo yes || echo no)"
 check "server-join: enables k3s-recovery" "yes" "$(grep -q 'rc-update add mulga-eks-k3s-recovery default' "${CALLS}" && echo yes || echo no)"
+check "server-join: starts k3s-recovery" "yes" "$(grep -q 'rc-service mulga-eks-k3s-recovery start' "${CALLS}" && echo yes || echo no)"
 # Join servers must NOT re-publish bootstrap; the first server already did.
 check "server-join: no k3s-first-boot" "no" "$(grep -q 'k3s-first-boot' "${CALLS}" && echo yes || echo no)"
 check "server-join: no k3s-agent" "no" "$(grep -q 'k3s-agent' "${CALLS}" && echo yes || echo no)"

scripts/images/eks-node/k3s-agent.initd

@@ -29,7 +29,7 @@ start_pre() {
     if [ -f /etc/spinifex-eks/agent.env ]; then
         # shellcheck disable=SC1091
         . /etc/spinifex-eks/agent.env
-        export K3S_URL K3S_TOKEN K3S_NODE_NAME K3S_NODE_LABEL
+        export K3S_URL K3S_TOKEN K3S_NODE_NAME
     fi
     if [ -z "${K3S_URL:-}" ] || [ -z "${K3S_TOKEN:-}" ]; then
         eerror "K3S_URL and K3S_TOKEN must be set in /etc/spinifex-eks/agent.env"

scripts/images/eks-node/k3s.initd

@@ -28,6 +28,16 @@ depend() {
 }
 
 start_pre() {
+    # mulga-eks-k3s-recovery drops this marker when a required-snapshot restore
+    # could not fetch its etcd snapshot. Refuse to start: a fresh DR seed with no
+    # snapshot would cluster-init an EMPTY datastore (silent total data loss). The
+    # marker is in /run tmpfs, cleared on reboot, so recovery retries the fetch
+    # next boot and k3s starts once the snapshot restores.
+    if [ -f /run/spinifex-eks/k3s-restore-block ]; then
+        eerror "restore-snapshot could not fetch the required etcd snapshot; refusing to start k3s on an empty datastore (see mulga-eks-k3s-recovery log)"
+        return 1
+    fi
+
     # config.yaml is written by k3s-first-boot from the skeleton + cloud-init
     # data. Refuse to start if neither exists.
     if [ ! -f /etc/rancher/k3s/config.yaml ]; then

scripts/images/eks-node/konnectivity/konnectivity-agent.yaml

@@ -9,7 +9,9 @@
 #
 # Runs on workers only: it carries no toleration for the control-plane
 # CriticalAddonsOnly=true:NoExecute taint, so it is never scheduled onto the CP
-# nodes (which run the server, not an agent).
+# nodes (which run the server, not an agent). It DOES tolerate the GPU
+# nvidia.com/gpu=present:NoSchedule taint — a GPU-only worker still needs
+# kubectl logs/exec against pods there, which routes through this agent.
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -39,6 +41,13 @@ spec:
     spec:
       serviceAccountName: konnectivity-agent
       priorityClassName: system-node-critical
+      # Without this, a GPU-only worker (tainted nvidia.com/gpu=present:NoSchedule)
+      # shows DESIRED=0 for this DaemonSet and kubectl logs/exec against pods
+      # scheduled there fails "No agent available".
+      tolerations:
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
       # hostNetwork so the agent reaches the NLB over the node network and
       # identifies itself to the server by the node's own IP (the address the
       # apiserver dials back through for kubelet/pod egress).

scripts/images/eks-node/manifest.conf

@@ -61,6 +61,7 @@ scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/00-rbac.yaml:/usr/share
 scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/10-controller.yaml:/usr/share/spinifex-eks/addons/aws-ebs-csi-driver/1.40.1/10-controller.yaml \
 scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/20-node.yaml:/usr/share/spinifex-eks/addons/aws-ebs-csi-driver/1.40.1/20-node.yaml \
 scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/30-storageclass.yaml:/usr/share/spinifex-eks/addons/aws-ebs-csi-driver/1.40.1/30-storageclass.yaml \
+scripts/images/eks-node/addons/nvidia-device-plugin/0.17.4/10-daemonset.yaml:/usr/share/spinifex-eks/addons/nvidia-device-plugin/0.17.4/10-daemonset.yaml \
 scripts/images/eks-node/coredns/coredns-mulga.yaml:/usr/share/spinifex-eks/coredns/coredns-mulga.yaml \
 scripts/images/eks-node/cloud-init-ec2.cfg:/etc/cloud/cloud.cfg.d/90-spinifex-ec2-imds.cfg \
 scripts/images/eks-node/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot \

scripts/images/eks-node/mulga-eks-etcd-snapshot.sh

@@ -66,14 +66,24 @@ fi
 #
 # Note: --aws-sigv4 requires curl >= 7.75 (Alpine 3.21 ships 8.x).
 URL="${SPINIFEX_PREDASTORE_ENDPOINT%/}/${BUCKET}/${KEY}"
-curl -fsSL \
+# Capture curl's own exit: piping straight into logger masks it behind logger's
+# (near-always 0), so a failed upload would log "uploaded" and delete the only
+# copy — snapshots would silently never land. Keep the local snapshot and exit
+# non-zero on failure so cron surfaces it and the next run still has data on disk.
+if UPLOAD_OUT=$(curl -fsSL \
     --aws-sigv4 "aws:amz:${AWS_REGION:-au-mel-1}:s3" \
     --user "${SPINIFEX_PREDASTORE_AKID}:${SPINIFEX_PREDASTORE_SECRET}" \
     -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
     --upload-file "${SNAPSHOT_FILE}" \
-    "${URL}" 2>&1 | logger -t mulga-eks-etcd-snapshot
-
-# Remove the local snapshot to bound disk usage; the canonical copy lives in
-# predastore. Restore path (`spx admin eks restore-snapshot`) pulls from there.
-rm -f "${SNAPSHOT_FILE}"
-logger -t mulga-eks-etcd-snapshot "uploaded ${KEY}"
+    "${URL}" 2>&1); then
+    [ -n "${UPLOAD_OUT}" ] && printf '%s\n' "${UPLOAD_OUT}" | logger -t mulga-eks-etcd-snapshot
+    # Remove the local snapshot only after a confirmed upload to bound disk usage;
+    # the canonical copy lives in predastore (`spx admin eks restore-snapshot`).
+    rm -f "${SNAPSHOT_FILE}"
+    logger -t mulga-eks-etcd-snapshot "uploaded ${KEY}"
+else
+    UPLOAD_RC=$?
+    [ -n "${UPLOAD_OUT}" ] && printf '%s\n' "${UPLOAD_OUT}" | logger -t mulga-eks-etcd-snapshot
+    logger -t mulga-eks-etcd-snapshot "ERROR: upload of ${KEY} failed (curl rc=${UPLOAD_RC}); keeping local snapshot ${SNAPSHOT_FILE}"
+    exit 1
+fi

scripts/images/eks-node/mulga-eks-etcd-snapshot_test.sh

@@ -29,10 +29,13 @@ done
 :
 EOF
 
-# curl stub: record the full argument line (carries --upload-file + URL).
+# curl stub: record the full argument line (carries --upload-file + URL). Exits
+# non-zero when CURL_FAIL=1 so the harness can exercise the failed-upload path.
 cat > "${STUBBIN}/curl" <<'EOF'
 #!/bin/sh
 echo "curl $*" >> "${CURL_CALLS}"
+[ "${CURL_FAIL:-0}" = "1" ] && { echo "curl: (7) connection refused" >&2; exit 7; }
+exit 0
 EOF
 
 # logger stub: swallow.
@@ -102,6 +105,24 @@ run server "${WORK}/15min/mulga-eks-etcd-snapshot"
 [ -s "${WORK}/curl.calls" ] && fail "no-env: must not upload" || pass "no-env: no upload without creds"
 ENVFILE="${ENVFILE_SAVE}"
 
+# --- Case 5: upload FAILS -> non-zero exit + local snapshot KEPT (not masked) ---
+# Set up like run() but expect a non-zero exit and assert the snapshot survives.
+K3S_CALLS="${WORK}/k3s.calls"; CURL_CALLS="${WORK}/curl.calls"
+SNAPSHOT_DIR="${WORK}/snapshots"; ROLE_FILE="${WORK}/role"; K3S_BIN=k3s
+: > "${K3S_CALLS}"; : > "${CURL_CALLS}"; rm -rf "${SNAPSHOT_DIR}"
+printf '%s' server > "${ROLE_FILE}"
+export K3S_CALLS CURL_CALLS SNAPSHOT_DIR ROLE_FILE K3S_BIN ENVFILE
+export TIER=frequent CURL_FAIL=1
+if sh "${WORK}/15min/mulga-eks-etcd-snapshot" </dev/null; then
+    fail "upload-fail: must exit non-zero when curl fails"
+else
+    pass "upload-fail: non-zero exit surfaces the failure"
+fi
+ls "${SNAPSHOT_DIR}"/etcd-frequent-* >/dev/null 2>&1 \
+    && pass "upload-fail: local snapshot kept for the next run" \
+    || fail "upload-fail: local snapshot deleted despite failed upload"
+unset TIER CURL_FAIL
+
 if [ "${FAILS}" -eq 0 ]; then
     echo "PASS: all mulga-eks-etcd-snapshot cases"
     exit 0

scripts/images/eks-node/mulga-eks-k3s-recovery.sh

@@ -7,10 +7,10 @@ set -eu
 # but embedded etcd never reformed quorum after a simultaneous restart).
 #
 # The reconciler cannot push into the guest, so recovery is pull-based: this
-# fetches a per-member directive epoch<TAB>action<TAB>snapshot through the AWS
-# gateway (eks-gateway-fetch -resource recovery), epoch-guarded so a directive
-# applies exactly once. Ordered `before k3s` so the etcd datastore is mutated
-# while the apiserver is stopped. Actions:
+# fetches a per-member directive epoch<TAB>action<TAB>snapshot<TAB>required
+# through the AWS gateway (eks-gateway-fetch -resource recovery), epoch-guarded
+# so a directive applies exactly once. Ordered `before k3s` so the etcd datastore
+# is mutated while the apiserver is stopped. Actions:
 #   none          steady state — nothing to do
 #   cluster-reset seed member: `k3s server --cluster-reset` reforms a
 #                 single-member etcd from this node's intact data (optionally
@@ -18,6 +18,13 @@ set -eu
 #   wipe-rejoin   follower: delete the local etcd datastore so k3s re-joins the
 #                 reset seed's quorum fresh on the next k3s start
 #
+# required=1 marks a snapshot that MUST restore (fresh DR seed via
+# `spx admin eks restore-snapshot`): a fresh seed has no intact local etcd, so a
+# cluster-reset without the snapshot would boot an EMPTY cluster. On a fetch
+# failure this drops BLOCK_MARKER and aborts boot; k3s.initd refuses to start on
+# the marker, so k3s never resets into an empty datastore. The fetch is retried
+# on the next boot (BLOCK_MARKER lives in /run tmpfs, cleared on reboot).
+#
 # Env knobs default to production paths; overridable so the logic is unit-testable
 # without root, IMDS, a gateway, or a real k3s.
 
@@ -25,6 +32,7 @@ ROLE_FILE=${ROLE_FILE:-/etc/spinifex-eks/role}
 ENVFILE=${ENVFILE:-/etc/spinifex-eks/first-boot.env}
 SNAPSHOT_ENVFILE=${SNAPSHOT_ENVFILE:-/etc/spinifex-eks/etcd-snapshot.env}
 EPOCH_FILE=${EPOCH_FILE:-/var/lib/spinifex-eks/recovery.epoch}
+BLOCK_MARKER=${BLOCK_MARKER:-/run/spinifex-eks/k3s-restore-block}
 K3S_BIN=${K3S_BIN:-/usr/local/bin/k3s}
 K3S_CONFIG=${K3S_CONFIG:-/etc/rancher/k3s/config.yaml}
 ETCD_DIR=${ETCD_DIR:-/var/lib/rancher/k3s/server/db/etcd}
@@ -72,12 +80,26 @@ fetch_snapshot() {
     [ -n "${SPINIFEX_PREDASTORE_SECRET:-}" ] || return 1
     _url="${SPINIFEX_PREDASTORE_ENDPOINT%/}/eks-backups-system/${EKS_ACCOUNT_ID}/${EKS_CLUSTER_NAME}/${_name}"
     mkdir -p "${SNAPSHOT_DIR}"
-    curl -fsSL \
+    # Capture curl's own exit — piping straight into logger would mask it behind
+    # logger's (near-always 0), so a failed download would look like success and
+    # the caller would wrongly cluster-reset from local etcd.
+    if _out=$(curl -fsSL \
         --aws-sigv4 "aws:amz:${AWS_REGION:-au-mel-1}:s3" \
         --user "${SPINIFEX_PREDASTORE_AKID}:${SPINIFEX_PREDASTORE_SECRET}" \
         -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
         -o "${SNAPSHOT_DIR}/${_name}" \
-        "${_url}" 2>&1 | logger -t "${LOGTAG}"
+        "${_url}" 2>&1); then
+        [ -n "${_out}" ] && printf '%s\n' "${_out}" | logger -t "${LOGTAG}"
+        return 0
+    else
+        # Capture rc as the first else statement: $? read after `fi` would be the
+        # if's own 0, re-masking the failure we are here to surface.
+        _rc=$?
+        [ -n "${_out}" ] && printf '%s\n' "${_out}" | logger -t "${LOGTAG}"
+        rm -f "${SNAPSHOT_DIR}/${_name}"
+        log "snapshot ${_name} download failed (curl rc=${_rc})"
+        return 1
+    fi
 }
 
 # Instance ID from IMDS keys the per-member directive. IMDSv2 token if enforced,
@@ -109,8 +131,11 @@ fi
 epoch=$(printf '%s' "${line}" | cut -f1)
 action=$(printf '%s' "${line}" | cut -f2)
 snapshot=$(printf '%s' "${line}" | cut -f3)
+snapshot_required=$(printf '%s' "${line}" | cut -f4)
 [ -n "${action}" ] || action=none
 case "${epoch}" in '' | *[!0-9]*) epoch=0 ;; esac
+# Absent 4th field (older gateway) reads as not-required, preserving HA-reform semantics.
+case "${snapshot_required}" in 1) snapshot_required=1 ;; *) snapshot_required=0 ;; esac
 
 if [ "${action}" = "none" ]; then
     log "directive action=none (epoch ${epoch}); nothing to do"
@@ -136,6 +161,13 @@ case "${action}" in
             if fetch_snapshot "${snapshot}"; then
                 restore_arg="--cluster-reset-restore-path=${SNAPSHOT_DIR}/${snapshot}"
                 log "cluster-reset restoring snapshot ${snapshot}"
+            elif [ "${snapshot_required}" = "1" ]; then
+                # Fresh DR seed: no intact local etcd, so a reset without the snapshot
+                # would boot an EMPTY cluster. Block k3s and abort; retry next boot.
+                log "ERROR: required snapshot ${snapshot} could not be fetched; aborting boot to avoid resetting into an EMPTY datastore (retries next boot)"
+                mkdir -p "$(dirname "${BLOCK_MARKER}")"
+                : > "${BLOCK_MARKER}"
+                exit 1
             else
                 log "WARN: snapshot ${snapshot} fetch failed; cluster-reset from local etcd only"
             fi

scripts/images/eks-node/mulga-eks-k3s-recovery_test.sh

@@ -24,7 +24,9 @@ printf '%s\n' "${DIRECTIVE_LINE}"
 EOF
 
 # curl stub: answers the IMDS token PUT + instance-id GET, and materialises a
-# snapshot file for any `-o <path>` download. Records every call.
+# snapshot file for any `-o <path>` download. Records every call. A snapshot
+# download (an `-o` arg) fails with a non-zero exit when CURL_SNAP_FAIL=1, so the
+# harness can exercise the fetch-failure path.
 cat > "${STUBBIN}/curl" <<'EOF'
 #!/bin/sh
 echo "curl $*" >> "${CURL_CALLS}"
@@ -36,7 +38,10 @@ for a in "$@"; do
 done
 prev=""
 for a in "$@"; do
-    [ "$prev" = "-o" ] && echo snapdata > "$a"
+    if [ "$prev" = "-o" ]; then
+        [ "${CURL_SNAP_FAIL:-0}" = "1" ] && { echo "curl: (22) 404" >&2; exit 22; }
+        echo snapdata > "$a"
+    fi
     prev="$a"
 done
 exit 0
@@ -80,7 +85,7 @@ pass() { echo "ok: $*"; }
 
 # run <role> <directive-line>: reset capture files + state, then run the script
 # with every path knob pointed into the temp dir.
-run() {
+_setup_run() {
     _role=$1
     DIRECTIVE_LINE=$2
     K3S_CALLS="${WORK}/k3s.calls"
@@ -89,13 +94,29 @@ run() {
     EPOCH_FILE="${WORK}/recovery.epoch"
     ETCD_DIR="${WORK}/etcd"
     SNAPSHOT_DIR="${WORK}/snapshots"
+    BLOCK_MARKER="${WORK}/run/k3s-restore-block"
     : > "${K3S_CALLS}"
     : > "${CURL_CALLS}"
+    rm -f "${BLOCK_MARKER}"
     printf '%s' "${_role}" > "${ROLE_FILE}"
-    export DIRECTIVE_LINE K3S_CALLS CURL_CALLS ROLE_FILE EPOCH_FILE ETCD_DIR SNAPSHOT_DIR
+    export DIRECTIVE_LINE K3S_CALLS CURL_CALLS ROLE_FILE EPOCH_FILE ETCD_DIR SNAPSHOT_DIR BLOCK_MARKER
     export ENVFILE SNAPSHOT_ENVFILE
     export K3S_BIN=k3s FETCH_BIN=eks-gateway-fetch
-    sh "${SCRIPT}" </dev/null || fail "role=${_role}: non-zero exit"
+}
+
+# run <role> <directive-line>: expects a zero exit (the steady-state path).
+run() {
+    _setup_run "$1" "$2"
+    sh "${SCRIPT}" </dev/null || fail "role=${1}: non-zero exit"
+}
+
+# run_abort <role> <directive-line>: expects a NON-zero exit (the required-snapshot
+# fetch-failure abort). CURL_SNAP_FAIL must be set by the caller.
+run_abort() {
+    _setup_run "$1" "$2"
+    if sh "${SCRIPT}" </dev/null; then
+        fail "role=${1}: expected non-zero exit (required-snapshot abort)"
+    fi
 }
 
 TAB=$(printf '\t')
@@ -149,6 +170,36 @@ grep -q 'aws-sigv4' "${WORK}/curl.calls" \
 [ -f "${WORK}/snapshots/etcd-daily-20260706T000000Z.snap" ] \
     && pass "snapshot: fetched into snapshot dir" || fail "snapshot: file not materialised"
 
+# --- Case 7: REQUIRED snapshot, fetch OK -> restore-path + no block marker ---
+rm -f "${WORK}/recovery.epoch"
+CURL_SNAP_FAIL=0; export CURL_SNAP_FAIL
+run server "5${TAB}cluster-reset${TAB}etcd-frequent-20260709T010000Z.snap${TAB}1"
+grep -q 'cluster-reset-restore-path=.*etcd-frequent-20260709T010000Z.snap' "${WORK}/k3s.calls" \
+    && pass "required-ok: restore-path passed" || fail "required-ok: no restore-path: $(cat "${WORK}/k3s.calls")"
+[ -f "${WORK}/run/k3s-restore-block" ] && fail "required-ok: must not block k3s on success" || pass "required-ok: no block marker"
+
+# --- Case 8: REQUIRED snapshot, fetch FAILS -> abort: block marker, no k3s, no epoch ---
+rm -f "${WORK}/recovery.epoch"
+CURL_SNAP_FAIL=1; export CURL_SNAP_FAIL
+run_abort server "6${TAB}cluster-reset${TAB}etcd-frequent-20260709T010000Z.snap${TAB}1"
+[ -f "${WORK}/run/k3s-restore-block" ] \
+    && pass "required-fail: block marker dropped to stop k3s" || fail "required-fail: no block marker"
+[ -s "${WORK}/k3s.calls" ] && fail "required-fail: must NOT cluster-reset into empty datastore" || pass "required-fail: k3s not reset"
+[ -f "${WORK}/recovery.epoch" ] && fail "required-fail: epoch must not record (retry next boot)" || pass "required-fail: epoch not recorded"
+
+# --- Case 9: NON-required snapshot, fetch FAILS -> local reset, no block, epoch recorded ---
+rm -f "${WORK}/recovery.epoch"
+CURL_SNAP_FAIL=1; export CURL_SNAP_FAIL
+run server "7${TAB}cluster-reset${TAB}etcd-frequent-20260709T010000Z.snap${TAB}0"
+grep -q 'server --config .* --cluster-reset' "${WORK}/k3s.calls" \
+    && pass "optional-fail: falls back to local cluster-reset" || fail "optional-fail: no reset: $(cat "${WORK}/k3s.calls")"
+grep -q 'cluster-reset-restore-path' "${WORK}/k3s.calls" \
+    && fail "optional-fail: must not pass a restore-path for a failed fetch" || pass "optional-fail: local-data reset only"
+[ -f "${WORK}/run/k3s-restore-block" ] && fail "optional-fail: must not block k3s (HA-reform semantics)" || pass "optional-fail: no block marker"
+[ "$(cat "${WORK}/recovery.epoch" 2>/dev/null)" = 7 ] \
+    && pass "optional-fail: epoch recorded" || fail "optional-fail: epoch not recorded"
+CURL_SNAP_FAIL=0; export CURL_SNAP_FAIL
+
 if [ "${FAILS}" -eq 0 ]; then
     echo "PASS: all mulga-eks-k3s-recovery cases"
     exit 0

scripts/images/eks-node/mulga-eks-state-report.sh

@@ -70,11 +70,21 @@ publish_report() {
     # Probe /healthz through the node's admin kubeconfig (apiserver runs
     # anonymous-auth=false, CIS): an unauthenticated curl would return 401.
     reason=
+    nodegroup_ready='{}'
     if kubectl get --raw='/healthz' 2>/dev/null | grep -q '^ok$'; then
         health=ok
         # Count Ready nodes only — k3s retains terminated workers as NotReady
         # until manually pruned, so a raw node count never drops on scale-down.
         node_count=$(kubectl get nodes --no-headers 2>/dev/null | awk '$2=="Ready"' | wc -l | tr -d ' ')
+        # Per-nodegroup Ready breakdown: -L appends the node's
+        # eks.amazonaws.com/nodegroup label as the last column ("<none>" when
+        # absent, e.g. the control-plane node itself). Grouping by that label —
+        # rather than only totalling — lets each nodegroup's own Ready-gate be
+        # scoped to its own workers instead of the cluster-wide count, so one
+        # nodegroup's Ready nodes can never mask another's stuck launch.
+        nodegroup_ready=$(kubectl get nodes --no-headers -L eks.amazonaws.com/nodegroup 2>/dev/null \
+            | awk '$2=="Ready" && $NF!="<none>"{c[$NF]++} END{sep=""; printf "{"; for (k in c) {printf "%s\"%s\":%d", sep, k, c[k]; sep=","}; printf "}"}')
+        [ -n "${nodegroup_ready}" ] || nodegroup_ready='{}'
     else
         health=fail
         node_count=0
@@ -91,11 +101,11 @@ publish_report() {
         } > "${CONSOLE:-/dev/console}" 2>&1 || true
     fi
     if [ -n "${reason}" ]; then
-        payload=$(printf '{"healthz":"%s","node_count":%s,"reason":"%s","ts":%s}' \
-            "${health}" "${node_count}" "${reason}" "$(date +%s)")
+        payload=$(printf '{"healthz":"%s","node_count":%s,"nodegroup_ready":%s,"reason":"%s","ts":%s}' \
+            "${health}" "${node_count}" "${nodegroup_ready}" "${reason}" "$(date +%s)")
     else
-        payload=$(printf '{"healthz":"%s","node_count":%s,"ts":%s}' \
-            "${health}" "${node_count}" "$(date +%s)")
+        payload=$(printf '{"healthz":"%s","node_count":%s,"nodegroup_ready":%s,"ts":%s}' \
+            "${health}" "${node_count}" "${nodegroup_ready}" "$(date +%s)")
     fi
     printf '%s' "${payload}" | eks-gateway-publish -channel state 2>&1 \
         | logger -t mulga-eks-state-report

scripts/images/eks-node/mulga-eks-state-report_test.sh

@@ -24,6 +24,7 @@ case "$*" in
     *"--raw=/healthz/etcd"*)   printf '%s\n' "${ETCD_BODY:-ok}" ;;
     *"--raw=/readyz?verbose"*) printf '%s\n' "${READYZ_BODY:-}" ;;
     *"--raw=/healthz"*)        printf '%s\n' "${HEALTHZ_BODY:-ok}" ;;
+    *"get nodes"*" -L "*)      printf '%s\n' "${NODES_LABELED_BODY:-}" ;;
     *"get nodes"*)             printf '%s\n' "${NODES_BODY:-}" ;;
     *) : ;;
 esac
@@ -75,15 +76,23 @@ run_agent() {
 }
 
 # --- Case 1: healthy apiserver, 3 Ready nodes -> no reason, quiet console ---
+# NODES_LABELED_BODY carries the -L eks.amazonaws.com/nodegroup breakdown: a
+# CP node ("<none>", excluded), 2 Ready ng-a workers, 1 Ready ng-b worker, and
+# a NotReady ng-b worker that must not count — proving the per-nodegroup tally
+# scopes to THAT nodegroup's own Ready nodes, not the cluster-wide total.
 HEALTHZ_BODY=ok
 NODES_BODY=$(printf 'ip-a Ready s\nip-b Ready s\nip-c Ready s\n')
+NODES_LABELED_BODY=$(printf 'ip-cp Ready s <none>\nip-a Ready s ng-a\nip-b Ready s ng-a\nip-c Ready s ng-b\nip-d NotReady s ng-b\n')
 ETCD_DB_DIR="${WORK}"
-export HEALTHZ_BODY NODES_BODY
+export HEALTHZ_BODY NODES_BODY NODES_LABELED_BODY
 run_agent
 P=$(cat "${WORK}/payload.json")
 case "${P}" in *'"healthz":"ok"'*) pass "healthy: healthz ok" ;; *) fail "healthy: healthz not ok: ${P}" ;; esac
 case "${P}" in *'"node_count":3'*) pass "healthy: node_count 3" ;; *) fail "healthy: node_count wrong: ${P}" ;; esac
 case "${P}" in *'"reason"'*) fail "healthy: reason must be absent: ${P}" ;; *) pass "healthy: no reason field" ;; esac
+case "${P}" in *'"nodegroup_ready":{'*) pass "healthy: nodegroup_ready object present" ;; *) fail "healthy: nodegroup_ready missing: ${P}" ;; esac
+case "${P}" in *'"ng-a":2'*) pass "healthy: ng-a Ready count scoped to its own 2 workers" ;; *) fail "healthy: ng-a count wrong: ${P}" ;; esac
+case "${P}" in *'"ng-b":1'*) pass "healthy: ng-b Ready count excludes its NotReady worker" ;; *) fail "healthy: ng-b count wrong: ${P}" ;; esac
 [ -s "${WORK}/console.out" ] && fail "healthy: console must be quiet" || pass "healthy: console quiet"
 
 # --- Case 2: unhealthy, etcd subcheck failing, etcd unreachable, disk ok ---
@@ -95,6 +104,7 @@ export HEALTHZ_BODY READYZ_BODY ETCD_BODY DF_AVAIL_KB
 run_agent
 P=$(cat "${WORK}/payload.json")
 case "${P}" in *'"healthz":"fail"'*) pass "unhealthy: healthz fail" ;; *) fail "unhealthy: healthz wrong: ${P}" ;; esac
+case "${P}" in *'"nodegroup_ready":{}'*) pass "unhealthy: nodegroup_ready empty (no kubectl node query on a failing apiserver)" ;; *) fail "unhealthy: nodegroup_ready wrong: ${P}" ;; esac
 case "${P}" in *'"reason":"readyz:[etcd poststarthook/start-service-ip-repair-controllers]; etcd:unreachable; disk:ok"'*) pass "unhealthy: reason names failing subchecks + etcd + disk" ;; *) fail "unhealthy: reason wrong: ${P}" ;; esac
 case "${P}" in *'"reason":"'*'"'*'"'*) : ;; esac
 C=$(cat "${WORK}/console.out")

scripts/images/ubuntu-gpu-nvidia.conf

@@ -10,6 +10,6 @@ IMAGE_SIZE="16G"
 
 SETUP_SCRIPT="scripts/images/ubuntu-gpu-nvidia-setup.sh"
 
-AMI_NAME="ubuntu-nvidia-gpu"
+AMI_NAME="ubuntu-26.04-nvidia-gpu-x86_64"
 
 SYSTEM_TAG="gpu-vendor=nvidia"

scripts/launch-gpu.sh

@@ -22,9 +22,9 @@ Usage:
 Options:
   --type TYPE    EC2 instance type to launch (required)
   --disk GB      Root disk size in GB (default: 50)
-  --ami  NAME    Override the AMI name to use (default: ubuntu-amd-gpu or ubuntu-nvidia-gpu)
+  --ami  NAME    Override the AMI name to use (default: ubuntu-amd-gpu or ubuntu-26.04-nvidia-gpu-x86_64)
   --amd          Force AMD GPU AMI  (ubuntu-amd-gpu)
-  --nvidia       Force NVIDIA GPU AMI (ubuntu-nvidia-gpu)
+  --nvidia       Force NVIDIA GPU AMI (ubuntu-26.04-nvidia-gpu-x86_64)
 
 Env overrides:
   SSH_KEY      Path to SSH private key (default: ~/.ssh/spinifex-key)
@@ -96,7 +96,7 @@ fi
 
 case "$GPU_BRAND" in
     amd)    AMI_NAME="ubuntu-amd-gpu" ;;
-    nvidia) AMI_NAME="ubuntu-nvidia-gpu" ;;
+    nvidia) AMI_NAME="ubuntu-26.04-nvidia-gpu-x86_64" ;;
 esac
 
 # --- Resolve AMI ---

spinifex/admin/admin.go

@@ -121,6 +121,10 @@ type PoolData struct {
 	PrefixLen  int      // Subnet prefix length (default 24)
 	DNSServers []string // DNS servers for VM DHCP (auto-detected from host)
 	DHCPMAC    string   // DHCP client MAC strategy: "derived" (default) or "interface"
+	// GwLrpRangeStart/End reserve gateway-LRP IPs for OVN routers. When empty
+	// the allocator auto-derives the top 16 host IPs of the pool subnet.
+	GwLrpRangeStart string
+	GwLrpRangeEnd   string
 }
 
 // PredastoreNodeConfig describes a single Predastore node for multi-node config generation.
@@ -181,20 +185,19 @@ func AWSGWServiceDNSNames(region, suffix string) []string {
 	}
 }
 
+// GenerateCertificatesIfNeeded prepares the TLS material for a node. The CA is
+// preserved across re-inits — it anchors trust for already-joined nodes, IPsec
+// peer certs, and CA-baked AMIs — so it is regenerated only when absent, never on
+// force. The CA-signed server cert is cheap to reissue, so force refreshes it to
+// pick up a changed bind IP / SANs while keeping the CA (and all trust) intact.
 func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string, awsRegion, internalSuffix string) (caCertPath string) {
 	caCertPath = filepath.Join(configDir, "ca.pem")
 	caKeyPath := filepath.Join(configDir, "ca.key")
 	serverCertPath := filepath.Join(configDir, "server.pem")
 	serverKeyPath := filepath.Join(configDir, "server.key")
 
-	// Check if we need to generate certificates
-	needsGeneration := force ||
-		!FileExists(caCertPath) || !FileExists(caKeyPath) ||
-		!FileExists(serverCertPath) || !FileExists(serverKeyPath)
-
-	if needsGeneration {
-		fmt.Println("\n🔐 Generating Certificate Authority and SSL certificates...")
-
+	if !FileExists(caCertPath) || !FileExists(caKeyPath) {
+		fmt.Println("\n🔐 Generating Certificate Authority...")
 		if err := GenerateCACert(caCertPath, caKeyPath); err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating CA certificate: %v\n", err)
 			os.Exit(1)
@@ -203,15 +206,6 @@ func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string, a
 		fmt.Printf("   CA Certificate: %s\n", caCertPath)
 		fmt.Printf("   CA Key: %s\n", caKeyPath)
 
-		extraDNS := AWSGWServiceDNSNames(awsRegion, internalSuffix)
-		if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, []string{bindIP}, extraDNS); err != nil {
-			fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
-			os.Exit(1)
-		}
-		fmt.Printf("✅ Server certificate generated (signed by CA):\n")
-		fmt.Printf("   Certificate: %s\n", serverCertPath)
-		fmt.Printf("   Key: %s\n", serverKeyPath)
-
 		// Print manual instructions only when not root (root gets auto-install)
 		if os.Getuid() != 0 {
 			fmt.Println("\n📋 To trust the Spinifex CA system-wide (recommended):")
@@ -220,7 +214,20 @@ func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string, a
 			fmt.Println("\n   This allows AWS CLI and other tools to trust Spinifex services automatically.")
 		}
 	} else {
-		fmt.Println("\n✅ CA and SSL certificates already exist")
+		fmt.Println("\n✅ Certificate Authority already exists (preserved)")
+	}
+
+	if force || !FileExists(serverCertPath) || !FileExists(serverKeyPath) {
+		extraDNS := AWSGWServiceDNSNames(awsRegion, internalSuffix)
+		if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, []string{bindIP}, extraDNS); err != nil {
+			fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("✅ Server certificate generated (signed by CA):\n")
+		fmt.Printf("   Certificate: %s\n", serverCertPath)
+		fmt.Printf("   Key: %s\n", serverKeyPath)
+	} else {
+		fmt.Println("✅ Server certificate already exists")
 	}
 
 	return caCertPath
@@ -500,9 +507,14 @@ func GenerateNATSToken() (string, error) {
 	return "nats_" + base64.URLEncoding.EncodeToString(bytes)[:32], nil
 }
 
+// certKeyBits is the RSA key size used when generating certificate keys.
+// It is a seam so tests can lower it for faster key generation; production
+// keeps the 4096-bit default.
+var certKeyBits = 4096
+
 // GenerateCACert generates a Certificate Authority certificate and key.
 func GenerateCACert(caCertPath, caKeyPath string) error {
-	caPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	caPrivateKey, err := rsa.GenerateKey(rand.Reader, certKeyBits)
 	if err != nil {
 		return fmt.Errorf("failed to generate CA private key: %w", err)
 	}
@@ -653,7 +665,7 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIP
 		return fmt.Errorf("CA key is not RSA")
 	}
 
-	serverPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	serverPrivateKey, err := rsa.GenerateKey(rand.Reader, certKeyBits)
 	if err != nil {
 		return fmt.Errorf("failed to generate server private key: %w", err)
 	}
@@ -754,8 +766,7 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIP
 
 // SetupAWSCredentials updates ~/.aws/credentials and ~/.aws/config.
 // When running under sudo, writes to SUDO_USER's home instead of root's.
-func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP string) error {
-	_ = wanIP
+func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP string) error {
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
 		return err
@@ -778,11 +789,16 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP s
 
 	profileName := "spinifex"
 
-	if err := UpdateAWSINIFile(credPath, profileName, map[string]string{
-		"aws_access_key_id":     accessKey,
-		"aws_secret_access_key": secretKey,
-	}); err != nil {
-		return err
+	// Empty credentials mean a --force re-init preserving the existing identity:
+	// the admin secret is not recoverable from disk and is unchanged, so leave the
+	// credentials file as-is and refresh only the config (endpoint/CA/region).
+	if accessKey != "" && secretKey != "" {
+		if err := UpdateAWSINIFile(credPath, profileName, map[string]string{
+			"aws_access_key_id":     accessKey,
+			"aws_secret_access_key": secretKey,
+		}); err != nil {
+			return err
+		}
 	}
 
 	configSection := profileName

spinifex/admin/admin_test.go

@@ -222,7 +222,7 @@ func TestSetupAWSCredentials_CreatesFiles(t *testing.T) {
 	dir := t.TempDir()
 	t.Setenv("HOME", dir)
 
-	err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/path/to/ca.pem", "", "")
+	err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/path/to/ca.pem", "")
 	require.NoError(t, err)
 
 	credData, _ := os.ReadFile(filepath.Join(dir, ".aws", "credentials"))
@@ -245,7 +245,7 @@ func TestSetupAWSCredentials_PreservesExistingProfiles(t *testing.T) {
 		"aws_access_key_id": "EXISTING_KEY",
 	}))
 
-	err := SetupAWSCredentials("NEWAKEY", "NEWSECRET", "us-west-2", "/ca.pem", "", "")
+	err := SetupAWSCredentials("NEWAKEY", "NEWSECRET", "us-west-2", "/ca.pem", "")
 	require.NoError(t, err)
 
 	data, _ := os.ReadFile(filepath.Join(awsDir, "credentials"))
@@ -258,7 +258,7 @@ func TestSetupAWSCredentials_UsesBindIP(t *testing.T) {
 	dir := t.TempDir()
 	t.Setenv("HOME", dir)
 
-	err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/ca.pem", "10.11.12.1", "")
+	err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/ca.pem", "10.11.12.1")
 	require.NoError(t, err)
 
 	configData, _ := os.ReadFile(filepath.Join(dir, ".aws", "config"))
@@ -270,13 +270,39 @@ func TestSetupAWSCredentials_FallsBackToLocalhostForWildcard(t *testing.T) {
 	dir := t.TempDir()
 	t.Setenv("HOME", dir)
 
-	err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/ca.pem", "0.0.0.0", "")
+	err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/ca.pem", "0.0.0.0")
 	require.NoError(t, err)
 
 	configData, _ := os.ReadFile(filepath.Join(dir, ".aws", "config"))
 	assert.Contains(t, string(configData), "https://localhost:9999")
 }
 
+// On a --force re-init the preserve path passes empty admin credentials: the
+// existing ~/.aws/credentials must be left intact while ~/.aws/config is still
+// refreshed (endpoint/CA for a changed bind IP).
+func TestSetupAWSCredentials_EmptyCredsRefreshesConfigOnly(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("HOME", dir)
+
+	awsDir := filepath.Join(dir, ".aws")
+	require.NoError(t, os.MkdirAll(awsDir, 0700))
+	require.NoError(t, UpdateAWSINIFile(filepath.Join(awsDir, "credentials"), "spinifex", map[string]string{
+		"aws_access_key_id":     "PRESERVED_KEY",
+		"aws_secret_access_key": "PRESERVED_SECRET",
+	}))
+
+	err := SetupAWSCredentials("", "", "ap-southeast-2", "/new/ca.pem", "10.11.12.5")
+	require.NoError(t, err)
+
+	credData, _ := os.ReadFile(filepath.Join(awsDir, "credentials"))
+	assert.Contains(t, string(credData), "PRESERVED_KEY", "existing admin credentials must be preserved on re-init")
+	assert.Contains(t, string(credData), "PRESERVED_SECRET")
+
+	configData, _ := os.ReadFile(filepath.Join(awsDir, "config"))
+	assert.Contains(t, string(configData), "https://10.11.12.5:9999", "config endpoint must be refreshed")
+	assert.Contains(t, string(configData), "/new/ca.pem")
+}
+
 // --- Certificate generation ---
 
 func TestGenerateCACert_CreatesValidCA(t *testing.T) {
@@ -617,12 +643,34 @@ func TestGenerateCertificatesIfNeeded(t *testing.T) {
 		assert.Equal(t, origModTime, caInfo2.ModTime())
 	})
 
-	t.Run("ForceRegenerates", func(t *testing.T) {
+	// --force must preserve the CA (trust anchor for joined nodes / baked AMIs)
+	// and only re-sign the server cert, which stays verifiable against that CA.
+	t.Run("ForcePreservesCARegeneratesServerCert", func(t *testing.T) {
 		origCA, _ := os.ReadFile(filepath.Join(dir, "ca.pem"))
+		origCAKey, _ := os.ReadFile(filepath.Join(dir, "ca.key"))
+		origServer, _ := os.ReadFile(filepath.Join(dir, "server.pem"))
+
+		GenerateCertificatesIfNeeded(dir, true, "10.9.8.7", "us-east-1", "spinifex.internal")
 
-		GenerateCertificatesIfNeeded(dir, true, "", "us-east-1", "spinifex.internal")
 		newCA, _ := os.ReadFile(filepath.Join(dir, "ca.pem"))
-		assert.NotEqual(t, origCA, newCA)
+		newCAKey, _ := os.ReadFile(filepath.Join(dir, "ca.key"))
+		newServer, _ := os.ReadFile(filepath.Join(dir, "server.pem"))
+		assert.Equal(t, origCA, newCA, "CA cert must be preserved on --force")
+		assert.Equal(t, origCAKey, newCAKey, "CA key must be preserved on --force")
+		assert.NotEqual(t, origServer, newServer, "server cert must be re-signed on --force")
+
+		// The re-signed server cert must verify against the preserved CA.
+		caBlock, _ := pem.Decode(newCA)
+		caCert, err := x509.ParseCertificate(caBlock.Bytes)
+		require.NoError(t, err)
+		pool := x509.NewCertPool()
+		pool.AddCert(caCert)
+
+		srvBlock, _ := pem.Decode(newServer)
+		srvCert, err := x509.ParseCertificate(srvBlock.Bytes)
+		require.NoError(t, err)
+		_, err = srvCert.Verify(x509.VerifyOptions{Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}})
+		assert.NoError(t, err, "re-signed server cert must verify against preserved CA")
 	})
 }
 

spinifex/admin/ipsec.go

@@ -60,7 +60,7 @@ func GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, hostname, nodeIP st
 		return err
 	}
 
-	peerKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	peerKey, err := rsa.GenerateKey(rand.Reader, certKeyBits)
 	if err != nil {
 		return fmt.Errorf("ipsec peer cert: generate key: %w", err)
 	}

spinifex/admin/main_test.go

@@ -0,0 +1,14 @@
+package admin
+
+import (
+	"os"
+	"testing"
+)
+
+// TestMain lowers the RSA key size for the whole admin test package. 2048-bit
+// keys are still valid and generate several times faster than the 4096-bit
+// production default, which dominates the admin suite runtime.
+func TestMain(m *testing.M) {
+	certKeyBits = 2048
+	os.Exit(m.Run())
+}

spinifex/admin/telemetry_test.go

@@ -63,7 +63,9 @@ func TestSendTelemetry_RespectsTimeout(t *testing.T) {
 	t.Parallel()
 
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(10 * time.Second)
+		// Outlive the 100ms client deadline so SendTelemetry aborts on its own
+		// context, but return quickly enough that server.Close() does not block.
+		time.Sleep(300 * time.Millisecond)
 		w.WriteHeader(http.StatusNoContent)
 	}))
 	defer server.Close()

spinifex/awsec2query/parser.go

@@ -7,6 +7,7 @@ import (
 	"reflect"
 	"strconv"
 	"strings"
+	"time"
 )
 
 // ErrSliceTooLarge is returned when a list exceeds maxSliceLen entries.
@@ -160,6 +161,16 @@ func setFieldValue(field reflect.Value, value string) error {
 			return err
 		}
 		field.SetBool(b)
+	case reflect.Struct:
+		if field.Type() == reflect.TypeFor[time.Time]() {
+			timestamp, err := time.Parse(time.RFC3339, value)
+			if err != nil {
+				return fmt.Errorf("parse RFC3339 timestamp: %w", err)
+			}
+			field.Set(reflect.ValueOf(timestamp))
+			return nil
+		}
+		return fmt.Errorf("unsupported struct type: %v", field.Type())
 	default:
 		return fmt.Errorf("unsupported field type: %v", field.Kind())
 	}

spinifex/awsec2query/parser_test.go

@@ -3,6 +3,7 @@ package awsec2query
 import (
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -649,3 +650,27 @@ func TestQueryParamsToStruct_SparseHugeIndexNoOp(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Empty(t, input.Filters)
 }
+
+func TestQueryParamsToStruct_CreateLaunchTemplateNestedData(t *testing.T) {
+	args := map[string]string{
+		"LaunchTemplateName":                                              "web-template",
+		"LaunchTemplateData.ImageId":                                      "ami-0abcdef1234567890",
+		"LaunchTemplateData.InstanceType":                                 "t3.micro",
+		"LaunchTemplateData.BlockDeviceMapping.1.DeviceName":              "/dev/sda1",
+		"LaunchTemplateData.BlockDeviceMapping.1.Ebs.VolumeSize":          "20",
+		"LaunchTemplateData.InstanceMarketOptions.SpotOptions.ValidUntil": "2026-07-10T12:00:00Z",
+	}
+
+	input := &ec2.CreateLaunchTemplateInput{}
+	err := QueryParamsToStruct(args, input)
+
+	require.NoError(t, err)
+	require.NotNil(t, input.LaunchTemplateData)
+	assert.Equal(t, "ami-0abcdef1234567890", aws.StringValue(input.LaunchTemplateData.ImageId))
+	require.Len(t, input.LaunchTemplateData.BlockDeviceMappings, 1)
+	assert.Equal(t, int64(20), aws.Int64Value(input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeSize))
+	require.NotNil(t, input.LaunchTemplateData.InstanceMarketOptions)
+	require.NotNil(t, input.LaunchTemplateData.InstanceMarketOptions.SpotOptions)
+	assert.Equal(t, time.Date(2026, time.July, 10, 12, 0, 0, 0, time.UTC),
+		*input.LaunchTemplateData.InstanceMarketOptions.SpotOptions.ValidUntil)
+}

spinifex/awserrors/awserrors.go

@@ -499,6 +499,17 @@ var (
 	ErrorELBv2IncompatibleProtocols        = "IncompatibleProtocols"
 	ErrorELBv2CertificateNotFound          = "CertificateNotFound"
 	ErrorELBv2SSLPolicyNotFound            = "SSLPolicyNotFound"
+
+	// Bedrock/bedrock-runtime error codes
+	ErrorValidationException = "ValidationException"
+	// ErrorResourceNotFoundException aliases ErrorEKSResourceNotFound: both
+	// services use the same wire code, so it is not a second ErrorLookup entry.
+	ErrorResourceNotFoundException   = ErrorEKSResourceNotFound
+	ErrorAccessDeniedException       = "AccessDeniedException"
+	ErrorThrottlingException         = "ThrottlingException"
+	ErrorModelNotReadyException      = "ModelNotReadyException"
+	ErrorServiceUnavailableException = "ServiceUnavailableException"
+	ErrorModelErrorException         = "ModelErrorException"
 )
 
 // ValidErrorCode returns the error code if it exists in ErrorLookup,
@@ -1021,4 +1032,13 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorELBv2IncompatibleProtocols:        {HTTPCode: 400, Message: "The listener protocol is incompatible with the target group protocol."},
 	ErrorELBv2CertificateNotFound:          {HTTPCode: 400, Message: "One or more specified certificates do not exist."},
 	ErrorELBv2SSLPolicyNotFound:            {HTTPCode: 400, Message: "The specified SSL policy does not exist."},
+
+	// Bedrock/bedrock-runtime error codes. ResourceNotFoundException reuses the
+	// EKS entry (ErrorEKSResourceNotFound) — same wire code, same HTTP status.
+	ErrorValidationException:         {HTTPCode: 400, Message: "The input fails to satisfy the constraints specified by the model or service."},
+	ErrorAccessDeniedException:       {HTTPCode: 403, Message: "You do not have sufficient access to perform this action."},
+	ErrorThrottlingException:         {HTTPCode: 429, Message: "The request was denied due to request throttling."},
+	ErrorModelNotReadyException:      {HTTPCode: 429, Message: "The model specified in the request is not ready to serve inference requests."},
+	ErrorServiceUnavailableException: {HTTPCode: 503, Message: "The service isn't currently available. Try again later."},
+	ErrorModelErrorException:         {HTTPCode: 424, Message: "The request failed because of an error while running the model."},
 }

spinifex/awserrors/awserrors_test.go

@@ -13,7 +13,7 @@ func TestErrorLookup_Structure(t *testing.T) {
 		t.Fatalf("ErrorLookup unexpectedly small: %d entries", len(ErrorLookup))
 	}
 
-	validHTTP := map[int]bool{400: true, 403: true, 404: true, 409: true, 412: true, 413: true, 500: true, 501: true, 503: true}
+	validHTTP := map[int]bool{400: true, 403: true, 404: true, 409: true, 412: true, 413: true, 424: true, 429: true, 500: true, 501: true, 503: true}
 	for code, msg := range ErrorLookup {
 		if !validHTTP[msg.HTTPCode] {
 			t.Errorf("%s has invalid HTTPCode %d", code, msg.HTTPCode)

spinifex/daemon/daemon.go

@@ -41,6 +41,7 @@ import (
 	handlers_ec2_image "github.com/mulgadc/spinifex/spinifex/handlers/ec2/image"
 	handlers_ec2_instance "github.com/mulgadc/spinifex/spinifex/handlers/ec2/instance"
 	handlers_ec2_key "github.com/mulgadc/spinifex/spinifex/handlers/ec2/key"
+	handlers_ec2_launchtemplate "github.com/mulgadc/spinifex/spinifex/handlers/ec2/launchtemplate"
 	handlers_ec2_natgw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/natgw"
 	handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup"
 	handlers_ec2_routetable "github.com/mulgadc/spinifex/spinifex/handlers/ec2/routetable"
@@ -132,6 +133,7 @@ type Daemon struct {
 	eigwService           *handlers_ec2_eigw.EgressOnlyIGWServiceImpl
 	igwService            *handlers_ec2_igw.IGWServiceImpl
 	placementGroupService *handlers_ec2_placementgroup.PlacementGroupServiceImpl
+	launchTemplateService *handlers_ec2_launchtemplate.LaunchTemplateServiceImpl
 	spotInstanceService   *handlers_ec2_spotinstance.SpotInstanceServiceImpl
 	vpcService            *handlers_ec2_vpc.VPCServiceImpl
 	eipService            handlers_ec2_eip.EIPService
@@ -175,9 +177,15 @@ type Daemon struct {
 	// initJetStream succeeds.
 	stateStore vm.StateStore
 
-	// Delay after QMP device_del before blockdev-del (default 1s, 0 in tests)
+	// Delay after QMP device_del before blockdev-del (default 1s, 0 in tests).
+	// Only used as a fallback when deviceDeletedTimeout is 0.
 	detachDelay time.Duration
 
+	// deviceDeletedTimeout bounds how long DetachVolume waits for QEMU's
+	// DEVICE_DELETED event after device_del before falling back to the
+	// blockdev-del retry loop (default 15s, 0 disables the wait in tests).
+	deviceDeletedTimeout time.Duration
+
 	// NATS connect retry options (nil uses defaults: 5min max, 500ms initial delay)
 	natsRetryOpts []utils.RetryOption
 
@@ -701,20 +709,21 @@ func NewDaemon(cfg *config.ClusterConfig) (*Daemon, error) {
 	}
 
 	d := &Daemon{
-		node:               cfg.Node,
-		clusterConfig:      cfg,
-		config:             &nodeCfg,
-		resourceMgr:        rm,
-		gpuProbe:           gpuProbe,
-		gpuManager:         gpuMgr,
-		ctx:                ctx,
-		cancel:             cancel,
-		vmMgr:              vm.NewManager(),
-		natsSubscriptions:  make(map[string]*nats.Subscription),
-		startTime:          time.Now(),
-		detachDelay:        1 * time.Second,
-		requireNATSTimeout: 30 * time.Second,
-		exitFunc:           os.Exit,
+		node:                 cfg.Node,
+		clusterConfig:        cfg,
+		config:               &nodeCfg,
+		resourceMgr:          rm,
+		gpuProbe:             gpuProbe,
+		gpuManager:           gpuMgr,
+		ctx:                  ctx,
+		cancel:               cancel,
+		vmMgr:                vm.NewManager(),
+		natsSubscriptions:    make(map[string]*nats.Subscription),
+		startTime:            time.Now(),
+		detachDelay:          1 * time.Second,
+		deviceDeletedTimeout: 15 * time.Second,
+		requireNATSTimeout:   30 * time.Second,
+		exitFunc:             os.Exit,
 	}
 	// Initialise peersReachable true so the first probe tick never fires a
 	// spurious reconcileOnHeal at startup. Mode() still requires natsConnected
@@ -799,6 +808,13 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.RemoveInstanceFromPlacementGroup", d.handleEC2RemoveInstanceFromPlacementGroup, "spinifex-workers"},
 		{"ec2.ReserveClusterNode", d.handleEC2ReserveClusterNode, "spinifex-workers"},
 		{"ec2.FinalizeClusterInstances", d.handleEC2FinalizeClusterInstances, "spinifex-workers"},
+		{"ec2.CreateLaunchTemplate", d.handleEC2CreateLaunchTemplate, "spinifex-workers"},
+		{"ec2.CreateLaunchTemplateVersion", d.handleEC2CreateLaunchTemplateVersion, "spinifex-workers"},
+		{"ec2.DeleteLaunchTemplate", d.handleEC2DeleteLaunchTemplate, "spinifex-workers"},
+		{"ec2.DeleteLaunchTemplateVersions", d.handleEC2DeleteLaunchTemplateVersions, "spinifex-workers"},
+		{"ec2.ModifyLaunchTemplate", d.handleEC2ModifyLaunchTemplate, "spinifex-workers"},
+		{"ec2.DescribeLaunchTemplates", d.handleEC2DescribeLaunchTemplates, "spinifex-workers"},
+		{"ec2.DescribeLaunchTemplateVersions", d.handleEC2DescribeLaunchTemplateVersions, "spinifex-workers"},
 		{"ec2.PutSpotInstanceRequests", d.handleEC2PutSpotInstanceRequests, "spinifex-workers"},
 		{"ec2.DescribeSpotInstanceRequests", d.handleEC2DescribeSpotInstanceRequests, "spinifex-workers"},
 		{"ec2.CancelSpotInstanceRequests", d.handleEC2CancelSpotInstanceRequests, "spinifex-workers"},
@@ -959,6 +975,7 @@ func (d *Daemon) subscribeAll() error {
 			natsSub{"eks.ListStagedAddonManifests", d.handleEKSListStagedAddonManifests, "spinifex-workers"},
 			natsSub{"eks.GetRecoveryDirective", d.handleEKSGetRecoveryDirective, "spinifex-workers"},
 			natsSub{"eks.SetRecoveryDirective", d.handleEKSSetRecoveryDirective, "spinifex-workers"},
+			natsSub{"eks.RestoreSnapshot", d.handleEKSRestoreSnapshot, "spinifex-workers"},
 			natsSub{"eks.AssociateIdentityProviderConfig", d.handleEKSAssociateIdentityProviderConfig, "spinifex-workers"},
 			natsSub{"eks.DescribeIdentityProviderConfig", d.handleEKSDescribeIdentityProviderConfig, "spinifex-workers"},
 			natsSub{"eks.ListIdentityProviderConfigs", d.handleEKSListIdentityProviderConfigs, "spinifex-workers"},
@@ -997,7 +1014,15 @@ func (d *Daemon) subscribeAll() error {
 			natsSub{"ecs.ListServices", d.handleECSListServices, "spinifex-workers"},
 			natsSub{"ecs.SubmitTaskStateChange", d.handleECSSubmitTaskStateChange, "spinifex-workers"},
 			natsSub{"ecs.PollAssignments", d.handleECSPollAssignments, "spinifex-workers"},
+			natsSub{"ecs.ReportTaskGPU", d.handleECSReportTaskGPU, "spinifex-workers"},
 			natsSub{"ecs.ProvisionCapacity", d.handleECSProvisionCapacity, "spinifex-workers"},
+			natsSub{"ecs.TagResource", d.handleECSTagResource, "spinifex-workers"},
+			natsSub{"ecs.UntagResource", d.handleECSUntagResource, "spinifex-workers"},
+			natsSub{"ecs.ListTagsForResource", d.handleECSListTagsForResource, "spinifex-workers"},
+			natsSub{"ecs.PutClusterCapacityProviders", d.handleECSPutClusterCapacityProviders, "spinifex-workers"},
+			natsSub{"ecs.CreateCapacityProvider", d.handleECSCreateCapacityProvider, "spinifex-workers"},
+			natsSub{"ecs.DescribeCapacityProviders", d.handleECSDescribeCapacityProviders, "spinifex-workers"},
+			natsSub{"ecs.DeleteCapacityProvider", d.handleECSDeleteCapacityProvider, "spinifex-workers"},
 		)
 	}
 
@@ -1211,6 +1236,8 @@ func (d *Daemon) assertNoClusterServicesInitialised() error {
 		return errors.New("d.igwService must be nil before startCluster")
 	case d.placementGroupService != nil:
 		return errors.New("d.placementGroupService must be nil before startCluster")
+	case d.launchTemplateService != nil:
+		return errors.New("d.launchTemplateService must be nil before startCluster")
 	case d.spotInstanceService != nil:
 		return errors.New("d.spotInstanceService must be nil before startCluster")
 	case d.vpcService != nil:
@@ -1255,6 +1282,12 @@ func (d *Daemon) startCluster() error {
 		return fmt.Errorf("initialize JetStream: %w", err)
 	}
 
+	// Set the default KV replica count before any handler creates a bucket, so
+	// lazily-created buckets are born at cluster-size replication instead of R1.
+	if d.clusterConfig != nil {
+		utils.SetDefaultKVReplicas(len(d.clusterConfig.Nodes))
+	}
+
 	// Remove the obsolete spinifex-dhcp-leases bucket (idempotent).
 	if js, jsErr := d.natsConn.JetStream(); jsErr == nil {
 		if err := utils.DeleteKVBucketIfExists(js, "spinifex-dhcp-leases"); err != nil {
@@ -1324,6 +1357,13 @@ func (d *Daemon) startCluster() error {
 		return fmt.Errorf("failed to initialize placement group service: %w", err)
 	}
 
+	d.launchTemplateService, err = initServiceWithRetry("launch template service", func() (*handlers_ec2_launchtemplate.LaunchTemplateServiceImpl, error) {
+		return handlers_ec2_launchtemplate.NewLaunchTemplateServiceImplWithNATS(d.config, d.natsConn)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to initialize launch template service: %w", err)
+	}
+
 	d.spotInstanceService, err = initServiceWithRetry("spot instance service", func() (*handlers_ec2_spotinstance.SpotInstanceServiceImpl, error) {
 		return handlers_ec2_spotinstance.NewSpotInstanceServiceImplWithNATS(d.config, d.natsConn)
 	})
@@ -1482,6 +1522,8 @@ func (d *Daemon) startCluster() error {
 			"err", err)
 	}
 
+	d.elbv2Service.StartLifecycleReaper(context.Background())
+
 	d.eksService, err = initServiceWithRetry("EKS service", func() (*handlers_eks.EKSServiceImpl, error) {
 		return handlers_eks.NewEKSServiceImpl(d.buildEKSServiceDeps())
 	})
@@ -1590,6 +1632,9 @@ func (d *Daemon) startCluster() error {
 		if eniRec := d.newENIReconciler(); eniRec != nil {
 			reapers = append(reapers, eniRec)
 		}
+		if gpuRec := d.newGPUPoolReconciler(); gpuRec != nil {
+			reapers = append(reapers, gpuRec)
+		}
 		if d.volumeService != nil {
 			reapers = append(reapers, d.volumeService.NewVolumeLeakReaper(d.leakedVolumeInstances))
 		}
@@ -2153,7 +2198,11 @@ func (d *Daemon) setupShutdown() {
 		for _, sub := range d.natsSubscriptions {
 			slog.Info("Unsubscribing from NATS", "subject", sub.Subject)
 			if err := sub.Unsubscribe(); err != nil {
-				slog.Error("Error unsubscribing from NATS", "err", err)
+				if errors.Is(err, nats.ErrBadSubscription) {
+					slog.Debug("NATS subscription already invalid during shutdown", "subject", sub.Subject)
+				} else {
+					slog.Error("Error unsubscribing from NATS", "err", err)
+				}
 			}
 		}
 
@@ -2219,18 +2268,26 @@ func (rm *ResourceManager) canAllocate(instanceType *ec2.InstanceTypeInfo, count
 // canAllocateLocked is the lock-free body of canAllocate. Caller must already
 // hold rm.mu for read or write. Extracted so allocate can re-check capacity
 // while holding the write lock without dropping it.
+//
+// GPU instance types (whole-GPU and MIG alike) are gated on BOTH the cpu/mem
+// calc AND GPU-slot availability. cpu/mem still applies to whole-GPU types —
+// the guest still consumes real host vCPU/RAM even though a GPU backs it —
+// and GPU-slot count is an additional hard constraint layered on top, so
+// admission never advertises more instances than there are GPU slots to
+// back them.
 func (rm *ResourceManager) canAllocateLocked(instanceType *ec2.InstanceTypeInfo, count int) int {
-	// Whole-GPU passthrough: gpuManager.Claim is the sole gate — one GPU per
-	// slot, so host CPU/memory is never the binding constraint.
-	// MIG types fall through to the normal check: one physical GPU backs many
-	// concurrent slices, each consuming real host CPU and memory, so those
-	// resources must be tracked and enforced like any non-GPU instance type.
 	instanceTypeName := ""
 	if instanceType.InstanceType != nil {
 		instanceTypeName = *instanceType.InstanceType
 	}
-	if instancetypes.IsGPUType(instanceType) && !instancetypes.IsMIGType(instanceTypeName) {
-		return count
+
+	requiresGPU := instancetypes.IsGPUType(instanceType)
+	availGPU := 0
+	if requiresGPU && rm.gpuManager != nil {
+		gpusNeeded := instancetypes.GPUCountForType(instanceTypeName)
+		if gpusNeeded > 0 {
+			availGPU = rm.gpuManager.Available() / gpusNeeded
+		}
 	}
 
 	n := canAllocateCount(
@@ -2239,7 +2296,7 @@ func (rm *ResourceManager) canAllocateLocked(instanceType *ec2.InstanceTypeInfo,
 		instanceTypeVCPUs(instanceType),
 		rm.instanceMemChargeMiB(instanceType),
 		count,
-		0, false,
+		availGPU, requiresGPU,
 	)
 	return rm.liveMemGate(n, instanceType)
 }

spinifex/daemon/daemon_gpu_handlers_test.go

@@ -0,0 +1,211 @@
+package daemon
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/gpu"
+	"github.com/mulgadc/spinifex/spinifex/types"
+	"github.com/mulgadc/spinifex/spinifex/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// gpuStatusRequest subscribes and requests handleNodeStatus on a fresh subject
+// so each test gets an isolated round trip.
+func gpuStatusRequest(t *testing.T, daemon *Daemon, subject string) types.NodeStatusResponse {
+	t.Helper()
+	sub, err := daemon.natsConn.Subscribe(subject, daemon.handleNodeStatus)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	reply, err := daemon.natsConn.Request(subject, nil, 5*time.Second)
+	require.NoError(t, err)
+
+	var resp types.NodeStatusResponse
+	require.NoError(t, json.Unmarshal(reply.Data, &resp))
+	return resp
+}
+
+// gpuVMsRequest subscribes and requests handleNodeVMs on a fresh subject.
+func gpuVMsRequest(t *testing.T, daemon *Daemon, subject string) types.NodeVMsResponse {
+	t.Helper()
+	sub, err := daemon.natsConn.Subscribe(subject, daemon.handleNodeVMs)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	reply, err := daemon.natsConn.Request(subject, nil, 5*time.Second)
+	require.NoError(t, err)
+
+	var resp types.NodeVMsResponse
+	require.NoError(t, json.Unmarshal(reply.Data, &resp))
+	return resp
+}
+
+func TestHandleNodeStatus_NoGPU(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+	daemon.gpuManager = nil
+
+	resp := gpuStatusRequest(t, daemon, "spinifex.node.status.nogpu")
+
+	assert.Empty(t, resp.GPUs)
+	assert.False(t, resp.GPUPassthrough)
+	assert.Equal(t, 0, resp.TotalGPUs)
+}
+
+func TestHandleNodeStatus_WholeGPU(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+
+	claimed := gpu.GPUDevice{PCIAddress: "0000:01:00.0", Model: "NVIDIA A10", MemoryMiB: 23028, IOMMUGroup: -1}
+	free := gpu.GPUDevice{PCIAddress: "0000:02:00.0", Model: "NVIDIA A10", MemoryMiB: 23028, IOMMUGroup: -1}
+	mgr := gpu.NewManager([]gpu.GPUDevice{claimed, free})
+	require.NoError(t, mgr.ReclaimByAddress(claimed.PCIAddress, "i-claimer"))
+	daemon.gpuManager = mgr
+
+	resp := gpuStatusRequest(t, daemon, "spinifex.node.status.wholegpu")
+
+	require.Len(t, resp.GPUs, 2)
+	byPCI := make(map[string]types.GPUInfo, 2)
+	for _, g := range resp.GPUs {
+		byPCI[g.PCIAddress] = g
+	}
+
+	claimedInfo := byPCI[claimed.PCIAddress]
+	assert.Equal(t, "NVIDIA A10", claimedInfo.Model)
+	assert.Equal(t, int64(23028), claimedInfo.VRAMMiB)
+	assert.False(t, claimedInfo.MIGEnabled)
+	assert.Empty(t, claimedInfo.MIGProfile)
+	assert.Nil(t, claimedInfo.Slices)
+	assert.Equal(t, "i-claimer", claimedInfo.InstanceID)
+
+	freeInfo := byPCI[free.PCIAddress]
+	assert.Empty(t, freeInfo.InstanceID)
+	assert.Nil(t, freeInfo.Slices)
+}
+
+func TestHandleNodeStatus_MIG(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+
+	mdevDir := t.TempDir()
+	mdev1 := filepath.Join(mdevDir, "uuid-gi1")
+	mdev2 := filepath.Join(mdevDir, "uuid-gi2")
+	require.NoError(t, os.MkdirAll(mdev1, 0o755))
+	require.NoError(t, os.MkdirAll(mdev2, 0o755))
+
+	dev := gpu.GPUDevice{
+		PCIAddress: "0000:03:00.0",
+		Model:      "NVIDIA RTX Pro 6000 Blackwell SE",
+		MemoryMiB:  98304,
+		IOMMUGroup: -1,
+		MIGCapable: true,
+		MIGEnabled: true,
+	}
+	mgr := gpu.NewManager(nil)
+	mgr.AddMIGInstances(dev, []gpu.MIGInstance{
+		{GIID: 1, MdevPath: mdev1, Profile: gpu.MIGProfile{Name: "1g.24gb", MemoryMiB: 24576}},
+		{GIID: 2, MdevPath: mdev2, Profile: gpu.MIGProfile{Name: "1g.24gb", MemoryMiB: 24576}},
+	})
+	require.NoError(t, mgr.ReclaimByMdev(mdev1, "i-slice-owner"))
+	daemon.gpuManager = mgr
+
+	resp := gpuStatusRequest(t, daemon, "spinifex.node.status.mig")
+
+	require.Len(t, resp.GPUs, 1)
+	g := resp.GPUs[0]
+	assert.Equal(t, dev.PCIAddress, g.PCIAddress)
+	assert.Equal(t, dev.Model, g.Model)
+	assert.True(t, g.MIGEnabled)
+	assert.Equal(t, "1g.24gb", g.MIGProfile)
+	assert.Empty(t, g.InstanceID, "whole-GPU InstanceID is unused in MIG mode")
+	require.Len(t, g.Slices, 2)
+
+	byGI := make(map[int]types.GPUSliceInfo, 2)
+	for _, s := range g.Slices {
+		byGI[s.GIID] = s
+	}
+	assert.Equal(t, "i-slice-owner", byGI[1].InstanceID)
+	assert.Equal(t, "1g.24gb", byGI[1].Profile)
+	assert.Equal(t, int64(24576), byGI[1].VRAMMiB)
+	assert.Equal(t, mdev1, byGI[1].MdevPath)
+	assert.Empty(t, byGI[2].InstanceID, "unclaimed slice is free")
+}
+
+func TestHandleNodeVMs_GPUAttachment_WholeGPU(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+	instanceType := getTestInstanceType(t)
+
+	dev := gpu.GPUDevice{PCIAddress: "0000:01:00.0", Model: "NVIDIA A10", MemoryMiB: 23028, IOMMUGroup: -1}
+	mgr := gpu.NewManager([]gpu.GPUDevice{dev})
+	require.NoError(t, mgr.ReclaimByAddress(dev.PCIAddress, "i-gpu-vm"))
+	daemon.gpuManager = mgr
+
+	daemon.vmMgr.Insert(&vm.VM{
+		ID:             "i-gpu-vm",
+		Status:         vm.StateRunning,
+		InstanceType:   instanceType,
+		GPUAttachments: []gpu.GPUAttachment{{PCIAddress: dev.PCIAddress}},
+	})
+
+	resp := gpuVMsRequest(t, daemon, "spinifex.node.vms.gpuwhole")
+
+	require.Len(t, resp.VMs, 1)
+	require.NotNil(t, resp.VMs[0].GPU)
+	assert.Equal(t, "NVIDIA A10", resp.VMs[0].GPU.Model)
+	assert.Equal(t, int64(23028), resp.VMs[0].GPU.VRAMMiB)
+	assert.Equal(t, dev.PCIAddress, resp.VMs[0].GPU.PCIAddress)
+	assert.Empty(t, resp.VMs[0].GPU.Profile)
+	assert.Empty(t, resp.VMs[0].GPU.MdevPath)
+}
+
+func TestHandleNodeVMs_GPUAttachment_MIG(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+	instanceType := getTestInstanceType(t)
+
+	mdevDir := t.TempDir()
+	mdevPath := filepath.Join(mdevDir, "uuid-gi1")
+	require.NoError(t, os.MkdirAll(mdevPath, 0o755))
+
+	dev := gpu.GPUDevice{PCIAddress: "0000:03:00.0", Model: "NVIDIA RTX Pro 6000 Blackwell SE", MemoryMiB: 98304}
+	mgr := gpu.NewManager(nil)
+	mgr.AddMIGInstances(dev, []gpu.MIGInstance{
+		{GIID: 1, MdevPath: mdevPath, Profile: gpu.MIGProfile{Name: "1g.24gb", MemoryMiB: 24576}},
+	})
+	require.NoError(t, mgr.ReclaimByMdev(mdevPath, "i-mig-vm"))
+	daemon.gpuManager = mgr
+
+	daemon.vmMgr.Insert(&vm.VM{
+		ID:             "i-mig-vm",
+		Status:         vm.StateRunning,
+		InstanceType:   instanceType,
+		GPUAttachments: []gpu.GPUAttachment{{MdevPath: mdevPath}},
+	})
+
+	resp := gpuVMsRequest(t, daemon, "spinifex.node.vms.gpumig")
+
+	require.Len(t, resp.VMs, 1)
+	require.NotNil(t, resp.VMs[0].GPU)
+	assert.Equal(t, dev.Model, resp.VMs[0].GPU.Model)
+	assert.Equal(t, int64(24576), resp.VMs[0].GPU.VRAMMiB)
+	assert.Equal(t, "1g.24gb", resp.VMs[0].GPU.Profile)
+	assert.Equal(t, mdevPath, resp.VMs[0].GPU.MdevPath)
+	assert.Empty(t, resp.VMs[0].GPU.PCIAddress)
+}
+
+func TestHandleNodeVMs_GPUAttachment_Absent(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+	instanceType := getTestInstanceType(t)
+
+	daemon.vmMgr.Insert(&vm.VM{
+		ID:           "i-no-gpu",
+		Status:       vm.StateRunning,
+		InstanceType: instanceType,
+	})
+
+	resp := gpuVMsRequest(t, daemon, "spinifex.node.vms.nogpu")
+
+	require.Len(t, resp.VMs, 1)
+	assert.Nil(t, resp.VMs[0].GPU)
+}

spinifex/daemon/daemon_handlers.go

@@ -6,14 +6,18 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"maps"
 	"net"
 	"net/http"
+	"slices"
 	"strconv"
 	"sync"
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/mulgadc/spinifex/spinifex/formation"
 	"github.com/mulgadc/spinifex/spinifex/gpu"
+	"github.com/mulgadc/spinifex/spinifex/network/host"
 	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/spinifex/spinifex/vm"
@@ -80,6 +84,10 @@ func handleNATSRequest[I any, O any](msg *nats.Msg, serviceFn func(context.Conte
 	}
 	output, err := serviceFn(ctx, input, accountID)
 	if err != nil {
+		// The error was otherwise only recorded on the OTel span, invisible
+		// without a trace backend. Log at Error so handler failures show up
+		// in the journal too.
+		slog.ErrorContext(ctx, "handleNATSRequest: service call failed", "subject", msg.Subject, "err", err)
 		utils.MarkSpanError(span, err)
 		respondWithError(msg, awserrors.ValidErrorCode(err.Error()))
 		return
@@ -307,11 +315,18 @@ func (d *Daemon) handleNodeStatus(msg *nats.Msg) {
 		InstanceTypes:  caps,
 	}
 
-	// Query service roles concurrently to halve worst-case latency (500ms vs 1s).
+	// Query service roles concurrently to keep worst-case latency bounded.
+	// OVN roles are probed only on DB quorum members; compute-only nodes skip
+	// the shell-out entirely.
 	var wg sync.WaitGroup
 	wg.Add(2)
 	go func() { defer wg.Done(); resp.NATSRole = d.queryNATSRole() }()
 	go func() { defer wg.Done(); resp.PredastoreRole = d.queryPredastoreRole() }()
+	if d.isOVNDBQuorumMember() {
+		wg.Add(2)
+		go func() { defer wg.Done(); resp.OVNNBRole = host.OVNDBRole(host.OVNNBTarget, host.OVNNBSchema) }()
+		go func() { defer wg.Done(); resp.OVNSBRole = host.OVNDBRole(host.OVNSBTarget, host.OVNSBSchema) }()
+	}
 	wg.Wait()
 
 	respondWithJSON(msg, resp)
@@ -345,6 +360,17 @@ func (d *Daemon) queryPredastoreRole() string {
 	return fetchPredastoreRole(url, roleTLSHTTPClient)
 }
 
+// isOVNDBQuorumMember reports whether this node hosts the clustered OVN NB/SB
+// databases. Compute-only nodes short-circuit OVN role probes to "" without
+// shelling out to ovn-appctl.
+func (d *Daemon) isOVNDBQuorumMember() bool {
+	if d.clusterConfig == nil {
+		return false
+	}
+	names := slices.Collect(maps.Keys(d.clusterConfig.Nodes))
+	return slices.Contains(formation.OVNDBQuorumNames(names), d.node)
+}
+
 var roleHTTPClient = &http.Client{Timeout: 500 * time.Millisecond}
 
 var roleTLSHTTPClient = &http.Client{

spinifex/daemon/daemon_handlers_ecs.go

@@ -112,8 +112,44 @@ func (d *Daemon) handleECSPollAssignments(msg *nats.Msg) {
 	handleNATSRequest(msg, d.ecsService.PollAssignments)
 }
 
+func (d *Daemon) handleECSReportTaskGPU(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecsService.ReportTaskGPU)
+}
+
 // --- Capacity ---
 
 func (d *Daemon) handleECSProvisionCapacity(msg *nats.Msg) {
 	handleNATSRequest(msg, d.ecsService.ProvisionCapacity)
 }
+
+// --- Tags ---
+
+func (d *Daemon) handleECSTagResource(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecsService.TagResource)
+}
+
+func (d *Daemon) handleECSUntagResource(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecsService.UntagResource)
+}
+
+func (d *Daemon) handleECSListTagsForResource(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecsService.ListTagsForResource)
+}
+
+// --- Capacity providers ---
+
+func (d *Daemon) handleECSPutClusterCapacityProviders(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecsService.PutClusterCapacityProviders)
+}
+
+func (d *Daemon) handleECSCreateCapacityProvider(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecsService.CreateCapacityProvider)
+}
+
+func (d *Daemon) handleECSDescribeCapacityProviders(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecsService.DescribeCapacityProviders)
+}
+
+func (d *Daemon) handleECSDeleteCapacityProvider(msg *nats.Msg) {
+	handleNATSRequest(msg, d.ecsService.DeleteCapacityProvider)
+}

spinifex/daemon/daemon_handlers_eks.go

@@ -130,6 +130,10 @@ func (d *Daemon) handleEKSSetRecoveryDirective(msg *nats.Msg) {
 	handleNATSRequest(msg, d.eksService.SetRecoveryDirective)
 }
 
+func (d *Daemon) handleEKSRestoreSnapshot(msg *nats.Msg) {
+	handleNATSRequest(msg, d.eksService.RestoreSnapshot)
+}
+
 // --- OIDC identity-provider configs ---
 
 func (d *Daemon) handleEKSAssociateIdentityProviderConfig(msg *nats.Msg) {

spinifex/daemon/daemon_handlers_instance.go

@@ -25,11 +25,9 @@ import (
 
 // startStoppedForwardTimeout bounds the wait for ec2.start.{nodeId} to respond.
 // StartStoppedInstance does volume rehydrate + QEMU launch + QMP handshake,
-// which can take 10-20s on a cold start. Match the awsgw upstream 30s budget
-// so a slow-but-alive target node never trips the fallback path — falling back
-// while the target is still launching causes a duplicate Run that collides on
-// the deterministic tap name (TUNSETIFF: Device or resource busy).
-const startStoppedForwardTimeout = 30 * time.Second
+// which can take 10-20s on a cold start. Match the awsgw upstream 30s budget.
+// A var (not const) so tests can shrink it instead of sleeping real seconds.
+var startStoppedForwardTimeout = 30 * time.Second
 
 // handleSetInstanceTags applies a create-tags/delete-tags mutation to a running
 // instance: central store first, then the record under the manager lock, so a
@@ -477,14 +475,11 @@ func (d *Daemon) handleEC2DescribeInstanceStatus(msg *nats.Msg) {
 
 // handleEC2StartStoppedInstance handles the generic ec2.start queue-group topic.
 // It reads the stopped instance's LastNode from shared KV and forwards the
-// request to ec2.start.{lastNode} when the instance last ran on a different
-// node. This keeps instances on their original node so the per-node resource
-// manager sees the correct allocation. Local fallback fires only when the
-// targeted node has no active subscriber (ErrNoResponders — node down or not
-// yet recovered) or returns InsufficientInstanceCapacity. A timeout from a
-// reachable-but-slow target is surfaced as ServerInternal so the caller can
-// retry; falling back in that case races the still-running launch on the
-// target and collides on the deterministic tap name.
+// request to ec2.start.{lastNode} to keep instances on their original node.
+// Local fallback fires on any forward failure — no responders, capacity, or
+// timeout — because StartStoppedInstance's ClaimStoppedInstance call is the
+// single cluster-wide serialization point, so a losing racer bails out
+// cleanly instead of double-starting.
 func (d *Daemon) handleEC2StartStoppedInstance(msg *nats.Msg) {
 	// Peek at the instance ID without full unmarshal — we only need it for the
 	// LastNode lookup. The full unmarshal happens inside StartStoppedInstance.
@@ -532,13 +527,11 @@ func (d *Daemon) handleEC2StartStoppedInstance(msg *nats.Msg) {
 			slog.Warn("ec2.start: original node has no subscriber, starting locally",
 				"instanceId", peek.InstanceID, "lastNode", lastNode, "err", err)
 		} else {
-			// Timeout or other transport error from a node whose subscription
-			// did exist. Do NOT fall back — the target may still be launching
-			// the VM, and a duplicate Run here would collide on tap setup.
-			slog.Error("ec2.start: forward to original node failed, not falling back",
+			// Timeout or other transport error — e.g. lastNode crashed without
+			// a clean NATS disconnect, so no ErrNoResponders fires. The atomic
+			// claim in StartStoppedInstance makes a local retry safe either way.
+			slog.Warn("ec2.start: forward to original node failed, attempting local start",
 				"instanceId", peek.InstanceID, "lastNode", lastNode, "err", err)
-			respondWithError(msg, awserrors.ErrorServerInternal)
-			return
 		}
 	}
 

spinifex/daemon/daemon_handlers_instance_tags_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/mulgadc/spinifex/spinifex/vm"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -52,6 +53,26 @@ func (m *memStoppedStore) DeleteStoppedInstance(id string) error {
 	return nil
 }
 
+// UpdateStoppedInstance mimics the real CAS semantics: a missing record
+// returns nats.ErrKeyNotFound instead of resurrecting it.
+func (m *memStoppedStore) UpdateStoppedInstance(id string, mutate func(*vm.VM)) (*vm.VM, error) {
+	v, ok := m.instances[id]
+	if !ok {
+		return nil, nats.ErrKeyNotFound
+	}
+	mutate(v)
+	return v, nil
+}
+
+func (m *memStoppedStore) ClaimStoppedInstance(id string) (*vm.VM, error) {
+	v, ok := m.instances[id]
+	if !ok {
+		return nil, vm.ErrStoppedInstanceClaimed
+	}
+	delete(m.instances, id)
+	return v, nil
+}
+
 func (m *memStoppedStore) WriteTerminatedInstance(string, *vm.VM) error { return nil }
 
 // tagTestDaemon returns a test daemon with an in-memory central tag store, an

spinifex/daemon/daemon_handlers_instance_test.go

@@ -36,6 +36,7 @@ type fakeStateStore struct {
 	stopped          map[string]*vm.VM
 	loadStoppedErr   error
 	writeStoppedErr  error
+	updateStoppedErr error
 	listStoppedErr   error
 	deleteStoppedErr error
 
@@ -44,6 +45,8 @@ type fakeStateStore struct {
 	deleteStoppedFailFirst bool
 	deleteStoppedAttempts  int
 
+	claimStoppedErr error
+
 	terminated         map[string]*vm.VM
 	writeTerminatedErr error
 	listTerminatedErr  error
@@ -104,6 +107,41 @@ func (f *fakeStateStore) DeleteStoppedInstance(id string) error {
 	return nil
 }
 
+// ClaimStoppedInstance mimics the real atomic-delete claim for tests: under
+// the store lock, remove and return the entry, or claimStoppedErr /
+// vm.ErrStoppedInstanceClaimed if it is already gone or a test forced an error.
+func (f *fakeStateStore) ClaimStoppedInstance(id string) (*vm.VM, error) {
+	if f.claimStoppedErr != nil {
+		return nil, f.claimStoppedErr
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	v, ok := f.stopped[id]
+	if !ok {
+		return nil, vm.ErrStoppedInstanceClaimed
+	}
+	delete(f.stopped, id)
+	return v, nil
+}
+
+// UpdateStoppedInstance mimics the real CAS semantics for tests: mutate runs
+// under the store lock against the stored value, and a missing record
+// returns nats.ErrKeyNotFound (matching JetStreamManager's createIfAbsent=false
+// behavior) rather than resurrecting it.
+func (f *fakeStateStore) UpdateStoppedInstance(id string, mutate func(*vm.VM)) (*vm.VM, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.updateStoppedErr != nil {
+		return nil, f.updateStoppedErr
+	}
+	v, ok := f.stopped[id]
+	if !ok {
+		return nil, nats.ErrKeyNotFound
+	}
+	mutate(v)
+	return v, nil
+}
+
 func (f *fakeStateStore) ListStoppedInstances() ([]*vm.VM, error) {
 	if f.listStoppedErr != nil {
 		return nil, f.listStoppedErr
@@ -147,6 +185,19 @@ func (f *fakeStateStore) DeleteTerminatedInstance(id string) error {
 	return nil
 }
 
+// UpdateTerminatedInstance mimics the real CAS semantics for tests: mutate
+// runs under the store lock against the stored value.
+func (f *fakeStateStore) UpdateTerminatedInstance(id string, mutate func(*vm.VM)) (*vm.VM, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	v, ok := f.terminated[id]
+	if !ok {
+		return nil, errors.New("terminated instance not found")
+	}
+	mutate(v)
+	return v, nil
+}
+
 var _ vm.StateStore = (*fakeStateStore)(nil)
 
 // daemonWithFakeStateStore returns a daemon wired with an in-memory NATS
@@ -257,6 +308,81 @@ func TestHandleEC2StartStoppedInstance_InstanceTypeUnknown(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, decodeError(t, reply.Data)["Code"])
 }
 
+// withShortForwardTimeout shrinks startStoppedForwardTimeout for the duration
+// of a test so a forced forward timeout doesn't cost real wall-clock seconds.
+func withShortForwardTimeout(t *testing.T, d time.Duration) {
+	t.Helper()
+	orig := startStoppedForwardTimeout
+	startStoppedForwardTimeout = d
+	t.Cleanup(func() { startStoppedForwardTimeout = orig })
+}
+
+// TestHandleEC2StartStoppedInstance_ForwardTimeoutFallsBackLocally pins
+// siv-481: a forward to LastNode that times out (as opposed to an immediate
+// ErrNoResponders) must still fall back to a local start attempt instead of
+// surfacing a bare ServerInternal. The target subscriber below is alive but
+// silent, so nats: timeout is the only error the forward can produce — proof
+// that the fallback path, not ErrNoResponders handling, is what fires here.
+// An unresolvable instance type turns "local start was attempted" into a
+// distinct, assertable response code (InsufficientInstanceCapacity) instead
+// of colliding with the old no-fallback ServerInternal response.
+func TestHandleEC2StartStoppedInstance_ForwardTimeoutFallsBackLocally(t *testing.T) {
+	withShortForwardTimeout(t, 50*time.Millisecond)
+
+	store := newFakeStateStore()
+	v := stoppedVMFixture("i-timeout-fallback", testAccountID)
+	v.InstanceType = "definitely.not.a.real.type"
+	v.LastNode = "node-other"
+	store.stopped[v.ID] = v
+	d := daemonWithFakeStateStore(t, store)
+
+	// Simulate a live-but-unresponsive original node: subscribed, so no
+	// ErrNoResponders, but it never replies, so the forward times out.
+	silentSub, err := d.natsConn.Subscribe("ec2.start.node-other", func(*nats.Msg) {})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = silentSub.Unsubscribe() })
+
+	body, _ := json.Marshal(handlers_ec2_instance.StartStoppedInstanceInput{InstanceID: v.ID})
+	reply := requestHandler(t, d.natsConn, "ec2.start.test5", d.handleEC2StartStoppedInstance, testAccountID, body)
+	assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, decodeError(t, reply.Data)["Code"],
+		"a forward timeout must fall back to a local start attempt, not a bare ServerInternal")
+}
+
+// TestHandleEC2StartStoppedInstance_ForwardTimeoutAfterRemoteClaim_NoDoubleStart
+// pins the other half of siv-481: if the forward times out on the caller's
+// side AFTER the original node already won the atomic claim (removed the
+// record from shared KV) and kept working, the caller's local fallback must
+// not double-start the instance. It should observe the record already gone
+// and fail cleanly, and must never insert a second copy into its own vmMgr.
+func TestHandleEC2StartStoppedInstance_ForwardTimeoutAfterRemoteClaim_NoDoubleStart(t *testing.T) {
+	withShortForwardTimeout(t, 50*time.Millisecond)
+
+	store := newFakeStateStore()
+	v := stoppedVMFixture("i-race-claim", testAccountID)
+	v.LastNode = "node-other"
+	store.stopped[v.ID] = v
+	d := daemonWithFakeStateStore(t, store)
+
+	// Simulate the original node winning the claim (atomically removing the
+	// shared-KV record, as StartStoppedInstance's ClaimStoppedInstance does)
+	// but never replying — e.g. still mid-launch when the caller's forward
+	// budget expires.
+	claimingSub, err := d.natsConn.Subscribe("ec2.start.node-other", func(*nats.Msg) {
+		_, claimErr := store.ClaimStoppedInstance(v.ID)
+		assert.NoError(t, claimErr)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = claimingSub.Unsubscribe() })
+
+	body, _ := json.Marshal(handlers_ec2_instance.StartStoppedInstanceInput{InstanceID: v.ID})
+	reply := requestHandler(t, d.natsConn, "ec2.start.test6", d.handleEC2StartStoppedInstance, testAccountID, body)
+
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, decodeError(t, reply.Data)["Code"],
+		"local fallback must see the record already claimed and fail without double-starting")
+	_, found := d.vmMgr.Get(v.ID)
+	assert.False(t, found, "local fallback must not insert a second running copy of an already-claimed instance")
+}
+
 // --- handleEC2TerminateStoppedInstance ---
 
 func TestHandleEC2TerminateStoppedInstance_LoadError(t *testing.T) {
@@ -370,7 +496,7 @@ func TestHandleEC2TerminateStoppedInstance_CrossTenantRejected(t *testing.T) {
 
 func TestHandleEC2ModifyInstanceAttribute_WriteFailureReturnsServerInternal(t *testing.T) {
 	store := newFakeStateStore()
-	store.writeStoppedErr = errors.New("kv write failed")
+	store.updateStoppedErr = errors.New("kv write failed")
 	v := stoppedVMFixture("i-mod-write-fail", testAccountID)
 	store.stopped[v.ID] = v
 	d := daemonWithFakeStateStore(t, store)

spinifex/daemon/daemon_handlers_launchtemplate.go

@@ -0,0 +1,31 @@
+package daemon
+
+import "github.com/nats-io/nats.go"
+
+func (d *Daemon) handleEC2CreateLaunchTemplate(msg *nats.Msg) {
+	handleNATSRequest(msg, d.launchTemplateService.CreateLaunchTemplate)
+}
+
+func (d *Daemon) handleEC2CreateLaunchTemplateVersion(msg *nats.Msg) {
+	handleNATSRequest(msg, d.launchTemplateService.CreateLaunchTemplateVersion)
+}
+
+func (d *Daemon) handleEC2DeleteLaunchTemplate(msg *nats.Msg) {
+	handleNATSRequest(msg, d.launchTemplateService.DeleteLaunchTemplate)
+}
+
+func (d *Daemon) handleEC2DeleteLaunchTemplateVersions(msg *nats.Msg) {
+	handleNATSRequest(msg, d.launchTemplateService.DeleteLaunchTemplateVersions)
+}
+
+func (d *Daemon) handleEC2ModifyLaunchTemplate(msg *nats.Msg) {
+	handleNATSRequest(msg, d.launchTemplateService.ModifyLaunchTemplate)
+}
+
+func (d *Daemon) handleEC2DescribeLaunchTemplates(msg *nats.Msg) {
+	handleNATSRequest(msg, d.launchTemplateService.DescribeLaunchTemplates)
+}
+
+func (d *Daemon) handleEC2DescribeLaunchTemplateVersions(msg *nats.Msg) {
+	handleNATSRequest(msg, d.launchTemplateService.DescribeLaunchTemplateVersions)
+}

spinifex/daemon/daemon_handlers_volume.go

@@ -59,6 +59,28 @@ func (d *Daemon) handleAttachVolume(ctx context.Context, msg *nats.Msg, command
 	}
 
 	if volCfg.VolumeMetadata.State != "available" {
+		if volCfg.VolumeMetadata.AttachedInstance == command.ID {
+			if command.AttachVolumeData.Device != "" &&
+				command.AttachVolumeData.Device != volCfg.VolumeMetadata.DeviceName {
+				slog.ErrorContext(ctx, "AttachVolume: requested device conflicts with existing attachment",
+					"volumeId", volumeID, "instanceId", command.ID,
+					"requestedDevice", command.AttachVolumeData.Device,
+					"attachedDevice", volCfg.VolumeMetadata.DeviceName)
+				// AWS returns VolumeInUse (not InvalidParameterValue) when a
+				// re-attach targets a device other than the one already in use.
+				respondWithError(msg, awserrors.ErrorVolumeInUse)
+				return
+			}
+
+			// Volume is already attached to this instance (e.g. a CSI
+			// ControllerPublishVolume retry after a slow first attach).
+			// Treat as an idempotent success instead of VolumeInUse.
+			slog.InfoContext(ctx, "AttachVolume: volume already attached to requesting instance, returning idempotent success",
+				"volumeId", volumeID, "instanceId", command.ID, "device", volCfg.VolumeMetadata.DeviceName)
+			d.respondWithVolumeAttachment(msg, volumeID, command.ID, volCfg.VolumeMetadata.DeviceName, "attached")
+			return
+		}
+
 		slog.ErrorContext(ctx, "AttachVolume: volume not available",
 			"volumeId", volumeID, "state", volCfg.VolumeMetadata.State)
 		respondWithError(msg, awserrors.ErrorVolumeInUse)
@@ -75,7 +97,7 @@ func (d *Daemon) handleAttachVolume(ctx context.Context, msg *nats.Msg, command
 		return
 	}
 
-	res, err := d.vmMgr.AttachVolume(ctx, instance.ID, volumeID, command.AttachVolumeData.Device)
+	device, err := d.vmMgr.AttachVolume(ctx, instance.ID, volumeID, command.AttachVolumeData.Device)
 	if err != nil {
 		respondWithError(msg, attachDetachErrorCode(err))
 		return
@@ -88,7 +110,7 @@ func (d *Daemon) handleAttachVolume(ctx context.Context, msg *nats.Msg, command
 	// A guest-form name here breaks the immediate post-attach wait loop.
 	// This diverges intentionally from BlockDeviceMappings, which retain
 	// the guest path under mulga-599.
-	d.respondWithVolumeAttachment(msg, volumeID, command.ID, res.DeviceName, "attached")
+	d.respondWithVolumeAttachment(msg, volumeID, command.ID, device, "attached")
 }
 
 // handleDetachVolume dispatches the QMP/state-machine pipeline to

spinifex/daemon/daemon_handlers_volume_test.go

@@ -1,13 +1,25 @@
 package daemon
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
+	"strings"
 	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awss3 "github.com/aws/aws-sdk-go/service/s3"
 
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/mulgadc/spinifex/spinifex/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/qmp"
+	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/mulgadc/spinifex/spinifex/vm"
+	"github.com/mulgadc/viperblock/viperblock"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 // TestAttachDetachErrorCode locks down the manager-error → AWS-API-code
@@ -86,3 +98,219 @@ func TestAttachDetachErrorCode(t *testing.T) {
 		})
 	}
 }
+
+// seedVolumeConfig writes a minimal VolumeConfig to config.json, matching
+// the seeding pattern used by the handleAttachVolume tests in
+// daemon_handlers_test.go.
+func seedVolumeConfig(t *testing.T, store *objectstore.MemoryObjectStore, volumeID string, meta viperblock.VolumeMetadata) {
+	t.Helper()
+	wrapper := struct {
+		VolumeConfig viperblock.VolumeConfig `json:"VolumeConfig"`
+	}{
+		VolumeConfig: viperblock.VolumeConfig{VolumeMetadata: meta},
+	}
+	data, err := json.Marshal(wrapper)
+	require.NoError(t, err)
+	_, err = store.PutObject(t.Context(), &awss3.PutObjectInput{
+		Bucket: aws.String("test-bucket"),
+		Key:    aws.String(volumeID + "/config.json"),
+		Body:   strings.NewReader(string(data)),
+	})
+	require.NoError(t, err)
+}
+
+// TestAttachVolume_IdempotentSameInstance verifies that re-attaching a
+// volume already attached to the requesting instance (e.g. a CSI
+// ControllerPublishVolume retry after a slow first attach) short-circuits
+// to an idempotent "attached" success instead of VolumeInUse, and does not
+// invoke vm.Manager.AttachVolume (the fake QMPClient on the seeded instance
+// has no live socket, so a real AttachVolume call would fail rather than
+// silently succeed with the pre-existing device).
+func TestAttachVolume_IdempotentSameInstance(t *testing.T) {
+	tests := []struct {
+		name            string
+		requestedDevice string
+	}{
+		{name: "no device specified", requestedDevice: ""},
+		{name: "device matches existing attachment", requestedDevice: "/dev/sdf"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			daemon, store := createFullTestDaemonWithStore(t, sharedNATSURL)
+
+			instanceID := "i-attach-idempotent-" + strings.ReplaceAll(tt.name, " ", "-")
+			volumeID := "vol-idempotent-" + strings.ReplaceAll(tt.name, " ", "-")
+
+			instance := &vm.VM{
+				ID:           instanceID,
+				Status:       vm.StateRunning,
+				AccountID:    testAccountID,
+				InstanceType: getTestInstanceType(t),
+				Instance:     &ec2.Instance{},
+				QMPClient:    &qmp.QMPClient{},
+			}
+			daemon.vmMgr.Insert(instance)
+
+			seedVolumeConfig(t, store, volumeID, viperblock.VolumeMetadata{
+				VolumeID:         volumeID,
+				SizeGiB:          10,
+				State:            "in-use",
+				TenantID:         testAccountID,
+				AttachedInstance: instanceID,
+				DeviceName:       "/dev/sdf",
+			})
+
+			sub, err := daemon.natsConn.Subscribe(
+				fmt.Sprintf("ec2.cmd.%s", instanceID),
+				daemon.handleEC2Events,
+			)
+			require.NoError(t, err)
+			defer sub.Unsubscribe()
+
+			command := types.EC2InstanceCommand{
+				ID: instanceID,
+				Attributes: types.EC2CommandAttributes{
+					AttachVolume: true,
+				},
+				AttachVolumeData: &types.AttachVolumeData{
+					VolumeID: volumeID,
+					Device:   tt.requestedDevice,
+				},
+			}
+			cmdData, err := json.Marshal(command)
+			require.NoError(t, err)
+
+			resp, err := natsRequest(daemon.natsConn,
+				fmt.Sprintf("ec2.cmd.%s", instanceID),
+				cmdData,
+				5*time.Second,
+			)
+			require.NoError(t, err)
+
+			var attachment ec2.VolumeAttachment
+			require.NoError(t, json.Unmarshal(resp.Data, &attachment))
+
+			assert.Equal(t, volumeID, *attachment.VolumeId)
+			assert.Equal(t, instanceID, *attachment.InstanceId)
+			assert.Equal(t, "/dev/sdf", *attachment.Device)
+			assert.Equal(t, "attached", *attachment.State)
+		})
+	}
+}
+
+// TestAttachVolume_InUseDifferentInstance is a regression test locking down
+// that a volume attached to a DIFFERENT instance than the requester still
+// returns VolumeInUse unchanged, distinguishing it from the same-instance
+// idempotent short-circuit.
+func TestAttachVolume_InUseDifferentInstance(t *testing.T) {
+	daemon, store := createFullTestDaemonWithStore(t, sharedNATSURL)
+
+	instanceID := "i-attach-vol-other-instance"
+	otherInstanceID := "i-already-attached-elsewhere"
+	volumeID := "vol-in-use-other-instance"
+
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateRunning,
+		AccountID:    testAccountID,
+		InstanceType: getTestInstanceType(t),
+		Instance:     &ec2.Instance{},
+		QMPClient:    &qmp.QMPClient{},
+	}
+	daemon.vmMgr.Insert(instance)
+
+	seedVolumeConfig(t, store, volumeID, viperblock.VolumeMetadata{
+		VolumeID:         volumeID,
+		SizeGiB:          10,
+		State:            "in-use",
+		TenantID:         testAccountID,
+		AttachedInstance: otherInstanceID,
+		DeviceName:       "/dev/sdf",
+	})
+
+	sub, err := daemon.natsConn.Subscribe(
+		fmt.Sprintf("ec2.cmd.%s", instanceID),
+		daemon.handleEC2Events,
+	)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	command := types.EC2InstanceCommand{
+		ID: instanceID,
+		Attributes: types.EC2CommandAttributes{
+			AttachVolume: true,
+		},
+		AttachVolumeData: &types.AttachVolumeData{
+			VolumeID: volumeID,
+		},
+	}
+	cmdData, err := json.Marshal(command)
+	require.NoError(t, err)
+
+	resp, err := natsRequest(daemon.natsConn,
+		fmt.Sprintf("ec2.cmd.%s", instanceID),
+		cmdData,
+		5*time.Second,
+	)
+	require.NoError(t, err)
+	assert.Contains(t, string(resp.Data), "VolumeInUse")
+}
+
+// TestAttachVolume_IdempotentSameInstance_DeviceMismatch verifies that a
+// same-instance re-attach requesting a DIFFERENT device than the one
+// already attached is treated as a real CSI conflict — AWS returns
+// VolumeInUse for this case — not silently echoed back or accepted.
+func TestAttachVolume_IdempotentSameInstance_DeviceMismatch(t *testing.T) {
+	daemon, store := createFullTestDaemonWithStore(t, sharedNATSURL)
+
+	instanceID := "i-attach-device-mismatch"
+	volumeID := "vol-device-mismatch"
+
+	instance := &vm.VM{
+		ID:           instanceID,
+		Status:       vm.StateRunning,
+		AccountID:    testAccountID,
+		InstanceType: getTestInstanceType(t),
+		Instance:     &ec2.Instance{},
+		QMPClient:    &qmp.QMPClient{},
+	}
+	daemon.vmMgr.Insert(instance)
+
+	seedVolumeConfig(t, store, volumeID, viperblock.VolumeMetadata{
+		VolumeID:         volumeID,
+		SizeGiB:          10,
+		State:            "in-use",
+		TenantID:         testAccountID,
+		AttachedInstance: instanceID,
+		DeviceName:       "/dev/sdf",
+	})
+
+	sub, err := daemon.natsConn.Subscribe(
+		fmt.Sprintf("ec2.cmd.%s", instanceID),
+		daemon.handleEC2Events,
+	)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	command := types.EC2InstanceCommand{
+		ID: instanceID,
+		Attributes: types.EC2CommandAttributes{
+			AttachVolume: true,
+		},
+		AttachVolumeData: &types.AttachVolumeData{
+			VolumeID: volumeID,
+			Device:   "/dev/sdg",
+		},
+	}
+	cmdData, err := json.Marshal(command)
+	require.NoError(t, err)
+
+	resp, err := natsRequest(daemon.natsConn,
+		fmt.Sprintf("ec2.cmd.%s", instanceID),
+		cmdData,
+		5*time.Second,
+	)
+	require.NoError(t, err)
+	assert.Contains(t, string(resp.Data), awserrors.ErrorVolumeInUse)
+}

spinifex/daemon/daemon_system_dispatch.go

@@ -202,8 +202,8 @@ func (d *Daemon) handleSystemTerminateInstance(msg *nats.Msg) {
 func (d *Daemon) serveSystemTerminateInstance(msg *nats.Msg) {
 	// Subject suffix is the instance ID; payload is unused but reserved for
 	// future flags. Tolerate empty payloads.
-	parts := splitSubjectTail(msg.Subject, "system.TerminateInstance.")
-	if parts == "" {
+	parts, ok := strings.CutPrefix(msg.Subject, "system.TerminateInstance.")
+	if !ok || parts == "" {
 		slog.Error("system.TerminateInstance: subject missing instance ID", "subject", msg.Subject)
 		respondWithSystemTerminateError(msg, awserrors.ErrorInvalidInstanceIDNotFound)
 		return
@@ -288,14 +288,3 @@ func respondWithSystemTerminateError(msg *nats.Msg, errMsg string) {
 		slog.Error("system.TerminateInstance: respond (error) failed", "err", err)
 	}
 }
-
-// splitSubjectTail returns the portion of subject after prefix, or empty
-// string if subject does not start with prefix. Helper kept local to avoid
-// touching utils for a one-liner.
-func splitSubjectTail(subject, prefix string) string {
-	tail, ok := strings.CutPrefix(subject, prefix)
-	if !ok {
-		return ""
-	}
-	return tail
-}

spinifex/daemon/daemon_system_dispatch_test.go

@@ -12,25 +12,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestSplitSubjectTail(t *testing.T) {
-	cases := []struct {
-		name    string
-		subject string
-		prefix  string
-		want    string
-	}{
-		{"happy", "system.TerminateInstance.i-abc", "system.TerminateInstance.", "i-abc"},
-		{"empty tail equals prefix length", "system.TerminateInstance.", "system.TerminateInstance.", ""},
-		{"prefix mismatch", "other.subject", "system.TerminateInstance.", ""},
-		{"shorter than prefix", "sys", "system.TerminateInstance.", ""},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			assert.Equal(t, tc.want, splitSubjectTail(tc.subject, tc.prefix))
-		})
-	}
-}
-
 func TestRespondWithSystemLaunchOutput(t *testing.T) {
 	nc, err := nats.Connect(sharedNATSURL)
 	require.NoError(t, err)

spinifex/daemon/daemon_system_instance.go

@@ -472,7 +472,7 @@ func (d *Daemon) launchAMISystemInstance(input *sysinstance.SystemInstanceInput)
 	// Management NIC must be set before LaunchRunInstances so the fw_cfg netcfg
 	// blob (built during launch) carries the static mgmt0 address.
 	if err := d.attachSystemMgmtNIC(inst); err != nil {
-		d.vmMgr.MarkFailed(inst, "mgmt_nic_setup_failed")
+		d.vmMgr.MarkFailed(context.Background(), inst, "mgmt_nic_setup_failed")
 		return nil, err
 	}
 
@@ -485,7 +485,7 @@ func (d *Daemon) launchAMISystemInstance(input *sysinstance.SystemInstanceInput)
 			extraAccount := resolveENIAccount(extra.AccountID, input.AccountID)
 			if _, attachErr := d.vpcService.AttachENI(extraAccount, extra.ENIID, inst.ID, int64(idx+1)); attachErr != nil {
 				slog.Error("launchAMISystemInstance: failed to attach extra ENI", "eniId", extra.ENIID, "instanceId", inst.ID, "err", attachErr)
-				d.vmMgr.MarkFailed(inst, "extra_eni_attach_failed")
+				d.vmMgr.MarkFailed(context.Background(), inst, "extra_eni_attach_failed")
 				return nil, fmt.Errorf("attach extra ENI %s: %w", extra.ENIID, attachErr)
 			}
 		}
@@ -684,7 +684,7 @@ func (d *Daemon) releaseSystemInstanceEIP(instance *vm.VM) {
 // state migration to terminated KV).
 func (d *Daemon) cleanupFailedSystemInstance(instance *vm.VM, _ *ec2.InstanceTypeInfo) {
 	d.releaseSystemInstanceEIP(instance)
-	d.vmMgr.MarkFailed(instance, "system_instance_launch_failed")
+	d.vmMgr.MarkFailed(context.Background(), instance, "system_instance_launch_failed")
 }
 
 // WaitForSystemInstance polls until the instance reaches running state or times out.
@@ -757,9 +757,17 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *handlers_elbv2.
 	//   i8042.no{pnp,aux,kbd}   — microvm has no PS/2; skip i8042 probe timeouts
 	var sb strings.Builder
 	sb.WriteString("console=ttyS0 quiet loglevel=3 mitigations=off tsc=reliable no_timer_check reboot=t i8042.nopnp i8042.noaux i8042.nokbd")
-	for i := range input.NICs {
+	// Slots must stay packed in the same order QEMU actually creates
+	// -device entries (buildNICNetdevs), so an unprovisioned mgmt NIC
+	// (skipped there) must not reserve a slot here either.
+	slot := 0
+	for i, nic := range input.NICs {
+		if !nicProvisioned(i, nic) {
+			continue
+		}
 		fmt.Fprintf(&sb, " virtio_mmio.device=0x200@0x%x:%d",
-			0xfeb00000+i*0x200, 5+i)
+			0xfeb00000+slot*0x200, 5+slot)
+		slot++
 	}
 	cmdline := sb.String()
 
@@ -805,11 +813,15 @@ func microvmMachineType() string {
 // buildNICNetdevs produces QEMU -netdev and -device entries for each NIC in
 // input. The NIC order determines the netdev IDs: net0 = primary ENI,
 // net1 = mgmt (if present), net2+ = extra ENIs. Extra ENIs beyond the primary
-// are only included when corresponding ExtraENIs entries exist.
+// are only included when corresponding ExtraENIs entries exist. A gap at
+// net1 (single-node hosts with no br-mgmt) is expected and fine for QEMU.
 func buildNICNetdevs(instanceID string, input *handlers_elbv2.SystemInstanceInput, machineType string) nicNetdevResult {
 	var res nicNetdevResult
 
 	for i, nic := range input.NICs {
+		if !nicProvisioned(i, nic) {
+			continue
+		}
 		netID := fmt.Sprintf("net%d", i)
 		// Resolve the tap name from the corresponding ENI.
 		tapName := tapNameForNIC(i, nic, instanceID, input)
@@ -822,6 +834,15 @@ func buildNICNetdevs(instanceID string, input *handlers_elbv2.SystemInstanceInpu
 	return res
 }
 
+// nicProvisioned reports whether NIC[i] has a real tap/device behind it.
+// The mgmt NIC (index 1) is only wired when br-mgmt allocated it a MAC at
+// launch (LaunchSystemInstance); single-node hosts without br-mgmt leave it
+// blank, so it must be excluded from netdevs, mmio slots, and guest config —
+// not just any empty-MAC NIC, only this specific, expected slot.
+func nicProvisioned(i int, nic handlers_elbv2.NICConfig) bool {
+	return i != 1 || nic.MAC != ""
+}
+
 // tapNameForNIC returns the Linux tap device name for a NIC at the given index.
 // Index 0 → primary ENI tap, index 1 → mgmt tap, index 2+ → extra ENI taps.
 func tapNameForNIC(idx int, _ handlers_elbv2.NICConfig, instanceID string, input *handlers_elbv2.SystemInstanceInput) string {
@@ -850,7 +871,17 @@ func (d *Daemon) writeFwCfgBlobs(instanceID string, input *handlers_elbv2.System
 	lbenvPath := filepath.Join(runtimeDir, fmt.Sprintf("fwcfg-%s-lbenv.tmp", instanceID))
 	cacertPath := filepath.Join(runtimeDir, fmt.Sprintf("fwcfg-%s-cacert.tmp", instanceID))
 
-	netcfg, err := buildNetcfgBlob(input.NICs)
+	// Exclude any unprovisioned mgmt NIC so the guest is only ever told about
+	// interfaces that physically exist — mirrors the netdev/mmio-slot skip
+	// in buildNICNetdevs/buildDirectBootConfig.
+	provisionedNICs := make([]handlers_elbv2.NICConfig, 0, len(input.NICs))
+	for i, nic := range input.NICs {
+		if nicProvisioned(i, nic) {
+			provisionedNICs = append(provisionedNICs, nic)
+		}
+	}
+
+	netcfg, err := buildNetcfgBlob(provisionedNICs)
 	if err != nil {
 		return nil, err
 	}

spinifex/daemon/daemon_system_instance_test.go

@@ -251,6 +251,87 @@ func TestBuildNICNetdevs_EmptyNICs(t *testing.T) {
 	assert.Empty(t, res.devices)
 }
 
+func TestBuildNICNetdevs_UnprovisionedMgmtNIC_Skipped(t *testing.T) {
+	// Single-node host with no br-mgmt: NIC[1] (mgmt) never got a MAC/tap.
+	input := &handlers_elbv2.SystemInstanceInput{
+		ENIID: "eni-000000000000000c",
+		NICs: []handlers_elbv2.NICConfig{
+			{MAC: "02:aa:bb:cc:dd:01", IsDefault: true},
+			{MAC: ""},
+		},
+	}
+	res := buildNICNetdevs("i-test3", input, microvmMachineType())
+	require.Len(t, res.netdevs, 1)
+	require.Len(t, res.devices, 1)
+	assert.Contains(t, res.netdevs[0].Value, "id=net0,")
+	assert.Contains(t, res.devices[0].Value, "netdev=net0")
+}
+
+func TestBuildNICNetdevs_ProvisionedMgmtNIC_Included(t *testing.T) {
+	// Multi-node host with br-mgmt: NIC[1] (mgmt) has a real MAC/tap.
+	input := &handlers_elbv2.SystemInstanceInput{
+		ENIID: "eni-000000000000000d",
+		NICs: []handlers_elbv2.NICConfig{
+			{MAC: "02:aa:bb:cc:dd:01", IsDefault: true},
+			{MAC: "02:aa:bb:cc:dd:02"},
+		},
+	}
+	res := buildNICNetdevs("i-test4", input, microvmMachineType())
+	require.Len(t, res.netdevs, 2)
+	require.Len(t, res.devices, 2)
+	assert.Contains(t, res.netdevs[0].Value, "id=net0,")
+	assert.Contains(t, res.netdevs[1].Value, "id=net1,")
+	assert.Contains(t, res.devices[1].Value, "netdev=net1")
+}
+
+func TestBuildNICNetdevs_UnprovisionedMgmtNIC_ExtraENIUnaffected(t *testing.T) {
+	// NIC[1] mgmt unprovisioned + NIC[2] extra ENI provisioned: net1 must be
+	// skipped but the extra ENI's tap name resolution (index-based) must not
+	// be shifted/renumbered.
+	input := &handlers_elbv2.SystemInstanceInput{
+		ENIID: "eni-primary000000000",
+		ExtraENIs: []handlers_elbv2.ExtraENIInput{
+			{ENIID: "eni-extra000000000000"},
+		},
+		NICs: []handlers_elbv2.NICConfig{
+			{MAC: "02:aa:bb:cc:dd:01", IsDefault: true},
+			{MAC: ""},
+			{MAC: "02:aa:bb:cc:dd:03"},
+		},
+	}
+	res := buildNICNetdevs("i-test5", input, microvmMachineType())
+	require.Len(t, res.netdevs, 2)
+	require.Len(t, res.devices, 2)
+	assert.Contains(t, res.netdevs[0].Value, "id=net0,")
+	// net1 (mgmt) skipped entirely — net2 (extra ENI) keeps its own ID.
+	assert.Contains(t, res.netdevs[1].Value, "id=net2,")
+	assert.Contains(t, res.devices[1].Value, "netdev=net2")
+	assert.Contains(t, res.devices[1].Value, "02:aa:bb:cc:dd:03")
+	// Extra ENI's tap name must still resolve against ExtraENIs[0], not be
+	// mis-mapped by the gap at index 1.
+	assert.Equal(t, vm.TapDeviceName("eni-extra000000000000"), tapNameForNIC(2, input.NICs[2], "i-test5", input))
+}
+
+// ---------------------------------------------------------------------------
+// nicProvisioned
+// ---------------------------------------------------------------------------
+
+func TestNicProvisioned_MgmtSlotEmptyMAC(t *testing.T) {
+	assert.False(t, nicProvisioned(1, handlers_elbv2.NICConfig{MAC: ""}))
+}
+
+func TestNicProvisioned_MgmtSlotWithMAC(t *testing.T) {
+	assert.True(t, nicProvisioned(1, handlers_elbv2.NICConfig{MAC: "02:aa:bb:cc:dd:02"}))
+}
+
+func TestNicProvisioned_NonMgmtSlotsAlwaysTrue(t *testing.T) {
+	// Only the mgmt slot (index 1) is ever gated on MAC presence — an empty
+	// MAC anywhere else (e.g. primary, not expected in practice) is not
+	// treated as "unprovisioned" by this guard.
+	assert.True(t, nicProvisioned(0, handlers_elbv2.NICConfig{MAC: ""}))
+	assert.True(t, nicProvisioned(2, handlers_elbv2.NICConfig{MAC: ""}))
+}
+
 // ---------------------------------------------------------------------------
 // microvmMachineType
 // ---------------------------------------------------------------------------
@@ -308,6 +389,65 @@ func TestWriteFwCfgBlobs_HappyPath(t *testing.T) {
 	}
 }
 
+func TestWriteFwCfgBlobs_UnprovisionedMgmtNIC_ExcludedFromGuestConfig(t *testing.T) {
+	// Single-node host with no br-mgmt: the mgmt NIC (index 1) never got a
+	// MAC/CIDR. The guest netcfg blob must not carry a NIC1 entry at all.
+	tmpDir := t.TempDir()
+	t.Setenv("XDG_RUNTIME_DIR", tmpDir)
+
+	d := &Daemon{}
+	instanceID := "i-fwcfg-nomgmt"
+	input := &handlers_elbv2.SystemInstanceInput{
+		NICs: []handlers_elbv2.NICConfig{
+			{MAC: "02:aa:bb:cc:dd:01", CIDR: "10.0.1.5/24", Gateway: "10.0.1.1", IsDefault: true},
+			{MAC: ""},
+		},
+	}
+
+	entries, err := d.writeFwCfgBlobs(instanceID, input)
+	require.NoError(t, err)
+	require.Len(t, entries, 3)
+
+	netcfgData, err := os.ReadFile(entries[0].File)
+	require.NoError(t, err)
+	assert.Contains(t, string(netcfgData), "NIC0_MAC=02:aa:bb:cc:dd:01")
+	// The unprovisioned mgmt NIC must not appear as NIC1 (nor anywhere else).
+	assert.NotContains(t, string(netcfgData), "NIC1_")
+	assert.NotContains(t, string(netcfgData), "MAC=\n")
+
+	for _, e := range entries {
+		assert.NoError(t, os.Remove(e.File))
+	}
+}
+
+func TestWriteFwCfgBlobs_ProvisionedMgmtNIC_IncludedInGuestConfig(t *testing.T) {
+	// Multi-node host with br-mgmt: mgmt NIC has a real MAC/CIDR and must
+	// still be wired into the guest config exactly as today.
+	tmpDir := t.TempDir()
+	t.Setenv("XDG_RUNTIME_DIR", tmpDir)
+
+	d := &Daemon{}
+	instanceID := "i-fwcfg-mgmt"
+	input := &handlers_elbv2.SystemInstanceInput{
+		NICs: []handlers_elbv2.NICConfig{
+			{MAC: "02:aa:bb:cc:dd:01", CIDR: "10.0.1.5/24", Gateway: "10.0.1.1", IsDefault: true},
+			{MAC: "02:aa:bb:cc:dd:02", CIDR: "192.168.100.5/24"},
+		},
+	}
+
+	entries, err := d.writeFwCfgBlobs(instanceID, input)
+	require.NoError(t, err)
+
+	netcfgData, err := os.ReadFile(entries[0].File)
+	require.NoError(t, err)
+	assert.Contains(t, string(netcfgData), "NIC0_MAC=02:aa:bb:cc:dd:01")
+	assert.Contains(t, string(netcfgData), "NIC1_MAC=02:aa:bb:cc:dd:02")
+
+	for _, e := range entries {
+		assert.NoError(t, os.Remove(e.File))
+	}
+}
+
 func TestWriteFwCfgBlobs_InvalidNICs_NoDefault(t *testing.T) {
 	tmpDir := t.TempDir()
 	t.Setenv("XDG_RUNTIME_DIR", tmpDir)
@@ -331,6 +471,73 @@ func TestWriteFwCfgBlobs_InvalidNICs_NoDefault(t *testing.T) {
 	}
 }
 
+// ---------------------------------------------------------------------------
+// buildDirectBootConfig — mmio slot packing
+// ---------------------------------------------------------------------------
+
+func TestBuildDirectBootConfig_MmioSlotsPacked_MgmtSkipped(t *testing.T) {
+	// NIC[1] (mgmt) unprovisioned (no br-mgmt) must not reserve an mmio slot;
+	// remaining NICs (primary, extra ENI) get packed, contiguous slots.
+	tmpDir := t.TempDir()
+	t.Setenv("XDG_RUNTIME_DIR", tmpDir)
+
+	d := &Daemon{resourceMgr: &ResourceManager{instanceTypes: map[string]*ec2.InstanceTypeInfo{}}}
+	input := &handlers_elbv2.SystemInstanceInput{
+		ENIID: "eni-mmio000000000001",
+		ExtraENIs: []handlers_elbv2.ExtraENIInput{
+			{ENIID: "eni-mmio-extra00000001"},
+		},
+		NICs: []handlers_elbv2.NICConfig{
+			{MAC: "02:aa:bb:cc:dd:01", IsDefault: true},
+			{MAC: ""},
+			{MAC: "02:aa:bb:cc:dd:03"},
+		},
+	}
+
+	cfg, err := d.buildDirectBootConfig("i-mmio-test", input)
+	require.NoError(t, err)
+
+	// Only 2 real devices (mgmt skipped) → slots packed at 0 and 1, not 0 and 2.
+	assert.Contains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00000:5")
+	assert.Contains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00200:6")
+	assert.NotContains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00400:7")
+
+	require.Len(t, cfg.NetDevs, 2)
+	require.Len(t, cfg.Devices, 2)
+
+	for _, e := range cfg.FwCfg {
+		assert.NoError(t, os.Remove(e.File))
+	}
+}
+
+func TestBuildDirectBootConfig_MmioSlotsUnchanged_MgmtProvisioned(t *testing.T) {
+	// Multi-node host with br-mgmt: all NICs provisioned, slots stay 1:1
+	// with NIC index as before this fix.
+	tmpDir := t.TempDir()
+	t.Setenv("XDG_RUNTIME_DIR", tmpDir)
+
+	d := &Daemon{resourceMgr: &ResourceManager{instanceTypes: map[string]*ec2.InstanceTypeInfo{}}}
+	input := &handlers_elbv2.SystemInstanceInput{
+		ENIID: "eni-mmio000000000002",
+		NICs: []handlers_elbv2.NICConfig{
+			{MAC: "02:aa:bb:cc:dd:01", IsDefault: true},
+			{MAC: "02:aa:bb:cc:dd:02"},
+		},
+	}
+
+	cfg, err := d.buildDirectBootConfig("i-mmio-test2", input)
+	require.NoError(t, err)
+
+	assert.Contains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00000:5")
+	assert.Contains(t, cfg.KernelCmdline, "virtio_mmio.device=0x200@0xfeb00200:6")
+	require.Len(t, cfg.NetDevs, 2)
+	require.Len(t, cfg.Devices, 2)
+
+	for _, e := range cfg.FwCfg {
+		assert.NoError(t, os.Remove(e.File))
+	}
+}
+
 // ---------------------------------------------------------------------------
 // refreshSystemInstanceState
 // ---------------------------------------------------------------------------

spinifex/daemon/daemon_test.go

@@ -119,7 +119,8 @@ func createTestDaemon(t *testing.T, natsURL string) *Daemon {
 	require.NoError(t, err, "Failed to connect to NATS")
 
 	daemon.natsConn = nc
-	daemon.detachDelay = 0 // Skip sleep in tests
+	daemon.detachDelay = 0          // Skip sleep in tests
+	daemon.deviceDeletedTimeout = 0 // Skip DEVICE_DELETED wait in tests
 
 	// Initialize services (needed for handler tests).
 	// jsManager is nil here; pass a nil literal to keep the StoppedInstanceStore
@@ -133,10 +134,11 @@ func createTestDaemon(t *testing.T, natsURL string) *Daemon {
 	// AttachVolume / DetachVolume manager methods enough plumbing to drive
 	// ebs.mount/unmount over NATS using the test's connection.
 	daemon.vmMgr.SetDeps(vm.Deps{
-		NodeID:             daemon.node,
-		VolumeMounter:      newVolumeMounterAdapter(daemon.natsConn, daemon.node, daemon.volumeService),
-		VolumeStateUpdater: daemon.volumeService,
-		DetachDelay:        daemon.detachDelay,
+		NodeID:               daemon.node,
+		VolumeMounter:        newVolumeMounterAdapter(daemon.natsConn, daemon.node, daemon.volumeService),
+		VolumeStateUpdater:   daemon.volumeService,
+		DetachDelay:          daemon.detachDelay,
+		DeviceDeletedTimeout: daemon.deviceDeletedTimeout,
 	})
 
 	t.Cleanup(func() {
@@ -745,7 +747,7 @@ func TestDaemon_BootAllocation(t *testing.T) {
 
 	// Pre-populate the local state file with test state. Post-1a, LoadState
 	// reads from the local file (KV is best-effort cache only).
-	err = WriteLocalState(daemon.localStatePath(), vms)
+	err = writeLocalState(daemon.localStatePath(), vms)
 	require.NoError(t, err)
 
 	// Manually trigger the LoadState and allocation logic normally found in Start()
@@ -2797,7 +2799,7 @@ func TestDaemon_WriteState_NilJSManager(t *testing.T) {
 
 func TestDaemon_LoadState_NilJSManager(t *testing.T) {
 	tmpDir := t.TempDir()
-	require.NoError(t, WriteLocalState(LocalStatePath(tmpDir), map[string]*vm.VM{
+	require.NoError(t, writeLocalState(LocalStatePath(tmpDir), map[string]*vm.VM{
 		"i-seed": {ID: "i-seed"},
 	}))
 
@@ -2984,16 +2986,47 @@ func TestGetAvailableInstanceTypeInfos_GPUType_ShowCapacity(t *testing.T) {
 	assert.Equal(t, 2, count12xl, "4 GPUs → 2 slots for g7e.12xlarge")
 }
 
-// canAllocate for GPU types returns count without CPU/memory gating.
-func TestCanAllocate_GPUType_BypassesCPUMemory(t *testing.T) {
+// canAllocate for GPU types is gated by CPU/memory like any other instance
+// type. Regression for mulga-jeo02: the whole-GPU early return used to
+// bypass this check entirely, letting a launch proceed on a host with no
+// memory headroom.
+func TestCanAllocate_GPUType_GatedByCPUMemory(t *testing.T) {
 	rm := &ResourceManager{
 		hostVCPU:      4,   // deliberately tiny
 		hostMemGB:     8.0, // deliberately tiny
 		instanceTypes: map[string]*ec2.InstanceTypeInfo{},
+		gpuManager:    gpu.NewManager([]gpu.GPUDevice{makeGPUDevice()}),
 	}
 	gpuType := makeGPUInstanceType("g7e.4xlarge", 16, 128*1024)
-	assert.Equal(t, 1, rm.canAllocate(gpuType, 1),
-		"GPU type must be allocatable even when CPU/memory would normally block it")
+	assert.Equal(t, 0, rm.canAllocate(gpuType, 1),
+		"GPU type must be blocked by cpu/mem exhaustion like any other instance type")
+}
+
+// canAllocate for GPU types succeeds once both cpu/mem headroom and a free
+// GPU slot exist.
+func TestCanAllocate_GPUType_SucceedsWithRoomAndSlot(t *testing.T) {
+	rm := &ResourceManager{
+		hostVCPU:      32,
+		hostMemGB:     256.0,
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{},
+		gpuManager:    gpu.NewManager([]gpu.GPUDevice{makeGPUDevice()}),
+	}
+	gpuType := makeGPUInstanceType("g7e.4xlarge", 16, 128*1024)
+	assert.Equal(t, 1, rm.canAllocate(gpuType, 1))
+}
+
+// canAllocate for GPU types is refused up front when the GPU pool has no
+// free slot, even with abundant cpu/mem headroom.
+func TestCanAllocate_GPUType_RefusedWhenGPUPoolExhausted(t *testing.T) {
+	rm := &ResourceManager{
+		hostVCPU:      32,
+		hostMemGB:     256.0,
+		instanceTypes: map[string]*ec2.InstanceTypeInfo{},
+		gpuManager:    gpu.NewManager(nil),
+	}
+	gpuType := makeGPUInstanceType("g7e.4xlarge", 16, 128*1024)
+	assert.Equal(t, 0, rm.canAllocate(gpuType, 1),
+		"no GPU in the pool must refuse admission regardless of cpu/mem headroom")
 }
 
 // --- GetSupportedInstanceTypeInfos ---
@@ -3275,6 +3308,48 @@ func TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable(t *testing.T) {
 		"a successful seal must still transition the volume to available")
 }
 
+// TestVolumeMounterAdapter_Unmount_NotFoundFlipsToAvailable verifies the
+// timeout-then-completed-seal retry path: an ebs.unmount response reporting
+// NotFound (the seal already completed on a prior, client-timed-out request)
+// must still flip a non-boot data volume to available, while the boot/EFI
+// gate stays intact.
+func TestVolumeMounterAdapter_Unmount_NotFoundFlipsToAvailable(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+	volState := &recordingVolState{}
+	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, volState)
+
+	sub, err := daemon.natsConn.Subscribe(adapter.topic("unmount"), func(msg *nats.Msg) {
+		var req types.EBSRequest
+		json.Unmarshal(msg.Data, &req)
+		resp := types.EBSUnMountResponse{
+			Volume:   req.Name,
+			NotFound: true,
+			Error:    fmt.Sprintf("Volume %s not found", req.Name),
+		}
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
+	})
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	inst := &vm.VM{
+		ID: "i-unmount-retry",
+		EBSRequests: types.EBSRequests{
+			Requests: []types.EBSRequest{
+				{Name: "vol-boot-retry", Boot: true, EFI: false},
+				{Name: "vol-data-retry", Boot: false, EFI: false},
+			},
+		},
+	}
+	require.NoError(t, adapter.Unmount(inst))
+
+	calls := volState.snapshot()
+	assert.Contains(t, calls, "vol-data-retry:available",
+		"a NotFound response (already-completed seal) must still flip a non-boot data volume to available")
+	assert.NotContains(t, calls, "vol-boot-retry:available",
+		"a boot volume must stay attached even when the seal retry reports NotFound")
+}
+
 // TestUnmountResponseError covers the shared seal-result parser used by both the
 // gated DetachVolume path (unmountOne) and the tolerated teardown path (Unmount).
 func TestUnmountResponseError(t *testing.T) {
@@ -3293,6 +3368,11 @@ func TestUnmountResponseError(t *testing.T) {
 	t.Run("malformed payload propagates", func(t *testing.T) {
 		require.Error(t, unmountResponseError([]byte("not json")))
 	})
+	t.Run("not found treated as idempotent success", func(t *testing.T) {
+		data, _ := json.Marshal(types.EBSUnMountResponse{Volume: "v", NotFound: true, Error: "Volume v not found"})
+		require.NoError(t, unmountResponseError(data),
+			"a NotFound response means the seal already completed on a prior request; a timeout-then-retry must not be treated as a failure")
+	})
 }
 
 // TestVolumeMounterAdapter_Mount_PartialFailureRollback verifies that when

spinifex/daemon/eks_deps.go

@@ -8,6 +8,7 @@ import (
 	handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup"
 	handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"log/slog"
 )
 
@@ -57,8 +58,10 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps {
 	}
 
 	internalSuffix := ""
+	clusterSize := 1
 	if d.clusterConfig != nil {
 		internalSuffix = d.clusterConfig.AWS.InternalSuffix
+		clusterSize = len(d.clusterConfig.Nodes)
 	}
 
 	gatewayCA := ""
@@ -72,32 +75,37 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps {
 	}
 
 	deps := handlers_eks.EKSServiceDeps{
-		Config:           d.config,
-		NATSConn:         d.natsConn,
-		MasterKey:        masterKey,
-		GatewayBaseURL:   d.resolveGatewayBaseURL(),
-		Region:           d.config.Region,
-		HolderID:         d.node,
-		InternalSuffix:   internalSuffix,
-		SystemGatewayURL: d.resolveSystemGatewayBaseURL(),
-		SystemAccessKey:  d.config.Predastore.AccessKey,
-		SystemSecretKey:  d.config.Predastore.SecretKey,
-		GatewayCACert:    gatewayCA,
-		VPCSG:            d.vpcService,
-		VPCK3s:           d.vpcService,
-		VPCSubnet:        &daemonEKSSubnetResolver{d: d},
-		NLB:              d.elbv2Service,
-		Instance:         d,
-		Image:            d.imageService,
-		EIP:              d.eipService,
-		IGW:              d.igwService,
-		Worker:           d,
-		VPCMgr:           d.vpcService,
-		NATGW:            d.natGatewayService,
-		RouteTable:       d.routeTableService,
-		PlacementGroup:   handlers_ec2_placementgroup.NewNATSPlacementGroupService(d.natsConn),
-		Scheduler:        handlers_eks.NewNATSHostScheduler(d.natsConn),
-		CPControl:        d.newEKSCPControl(),
+		Config:              d.config,
+		NATSConn:            d.natsConn,
+		MasterKey:           masterKey,
+		GatewayBaseURL:      d.resolveGatewayBaseURL(),
+		Region:              d.config.Region,
+		HolderID:            d.node,
+		ClusterSize:         clusterSize,
+		InternalSuffix:      internalSuffix,
+		SystemGatewayURL:    d.resolveSystemGatewayBaseURL(),
+		SystemAccessKey:     d.config.Predastore.AccessKey,
+		SystemSecretKey:     d.config.Predastore.SecretKey,
+		GatewayCACert:       gatewayCA,
+		SystemPredastoreURL: d.resolveSystemPredastoreURL(),
+		SnapshotStore: objectstore.NewS3ObjectStoreFromConfig(
+			d.config.Predastore.Host, d.config.Predastore.Region,
+			d.config.Predastore.AccessKey, d.config.Predastore.SecretKey),
+		VPCSG:          d.vpcService,
+		VPCK3s:         d.vpcService,
+		VPCSubnet:      &daemonEKSSubnetResolver{d: d},
+		NLB:            d.elbv2Service,
+		Instance:       d,
+		Image:          d.imageService,
+		EIP:            d.eipService,
+		IGW:            d.igwService,
+		Worker:         d,
+		VPCMgr:         d.vpcService,
+		NATGW:          d.natGatewayService,
+		RouteTable:     d.routeTableService,
+		PlacementGroup: handlers_ec2_placementgroup.NewNATSPlacementGroupService(d.natsConn),
+		Scheduler:      handlers_eks.NewNATSHostScheduler(d.natsConn),
+		CPControl:      d.newEKSCPControl(),
 	}
 
 	// A KV-backed IAM service (sharing the gateway's buckets over NATS) lets EKS
@@ -173,3 +181,25 @@ func (d *Daemon) resolveSystemGatewayBaseURL() string {
 	}
 	return "https://" + net.JoinHostPort(d.mgmtBridgeIP, gatewayPort)
 }
+
+// resolveSystemPredastoreURL derives the predastore URL a system instance (the
+// K3s server VM) fetches/PUTs etcd snapshots to. Same mgmt-bridge-only
+// reachability constraint as resolveSystemGatewayBaseURL: config.Predastore.Host
+// is rewritten to 127.0.0.1 for the local daemon, which a guest cannot dial, so
+// the bridge IP is substituted with the configured port. Falls back to the
+// configured Predastore.Host (scheme-prefixed) when no mgmt bridge exists.
+func (d *Daemon) resolveSystemPredastoreURL() string {
+	predastorePort := "8443"
+	if d.config.Predastore.Host != "" {
+		if _, port, splitErr := net.SplitHostPort(d.config.Predastore.Host); splitErr == nil && port != "" {
+			predastorePort = port
+		}
+	}
+	if d.mgmtBridgeIP == "" {
+		if d.config.Predastore.Host == "" {
+			return ""
+		}
+		return "https://" + d.config.Predastore.Host
+	}
+	return "https://" + net.JoinHostPort(d.mgmtBridgeIP, predastorePort)
+}

spinifex/daemon/eks_deps_test.go

@@ -84,3 +84,74 @@ func TestResolveGatewayHost(t *testing.T) {
 		})
 	}
 }
+
+// TestResolveSystemPredastoreURL covers the mgmt-bridge-present,
+// mgmt-bridge-absent, and no-config branches: a guest system VM (the K3s
+// server) can only reach predastore via the mgmt bridge, so the local
+// loopback-rewritten Predastore.Host must never leak into the URL handed to
+// a guest.
+func TestResolveSystemPredastoreURL(t *testing.T) {
+	tests := []struct {
+		name           string
+		predastoreHost string
+		mgmtBridge     string
+		want           string
+	}{
+		{
+			name:           "mgmt_bridge_present_uses_bridge_ip_and_configured_port",
+			predastoreHost: "127.0.0.1:9443",
+			mgmtBridge:     "10.15.8.1",
+			want:           "https://10.15.8.1:9443",
+		},
+		{
+			name:           "mgmt_bridge_present_defaults_port_when_host_has_none",
+			predastoreHost: "predastore-host-no-port",
+			mgmtBridge:     "10.15.8.1",
+			want:           "https://10.15.8.1:8443",
+		},
+		{
+			name:           "no_mgmt_bridge_falls_back_to_configured_host",
+			predastoreHost: "predastore.internal:8443",
+			want:           "https://predastore.internal:8443",
+		},
+		{
+			name: "no_mgmt_bridge_no_host_returns_empty",
+			want: "",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			d := &Daemon{
+				mgmtBridgeIP: tc.mgmtBridge,
+				config: &config.Config{
+					Predastore: config.PredastoreConfig{Host: tc.predastoreHost},
+				},
+			}
+			assert.Equal(t, tc.want, d.resolveSystemPredastoreURL())
+		})
+	}
+}
+
+// TestBuildEKSServiceDeps pins that the mgmt-bridge-derived predastore URL and
+// a non-nil snapshot object store actually reach EKSServiceDeps — RestoreSnapshot
+// silently can't resolve "latest snapshot" without SnapshotStore, and CP launch
+// silently can't write etcd-snapshot.env without SystemPredastoreURL.
+func TestBuildEKSServiceDeps(t *testing.T) {
+	d := &Daemon{
+		mgmtBridgeIP: "10.15.8.1",
+		config: &config.Config{
+			Predastore: config.PredastoreConfig{
+				Host:      "127.0.0.1:8443",
+				AccessKey: "AKIAPREDASTORE",
+				SecretKey: "pred-s3cr3t",
+				Region:    "us-east-1",
+			},
+		},
+	}
+
+	deps := d.buildEKSServiceDeps()
+
+	assert.Equal(t, "https://10.15.8.1:8443", deps.SystemPredastoreURL)
+	assert.NotNil(t, deps.SnapshotStore)
+}

spinifex/daemon/eks_worker_launch.go

@@ -22,6 +22,9 @@ import (
 // Compile-time check that Daemon satisfies the EKS worker-launch surface.
 var _ handlers_eks.WorkerLauncher = (*Daemon)(nil)
 
+// terminateRetrySleep is the backoff seam between NoResponders retries; tests override it.
+var terminateRetrySleep = time.Sleep
+
 // RunWorkerInstance launches a nodegroup worker on the local node.
 func (d *Daemon) RunWorkerInstance(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
 	return d.RunWorkerInstanceOnNode(ctx, "", input, accountID)
@@ -50,7 +53,7 @@ func (d *Daemon) RunWorkerInstanceOnNode(ctx context.Context, nodeID string, inp
 		return nil, errors.New("eks worker: NATS connection not initialized")
 	}
 	subject := fmt.Sprintf("ec2.RunInstances.%s.%s", aws.StringValue(input.InstanceType), nodeID)
-	return utils.NATSRequestCtx[ec2.Reservation](ctx, d.natsConn, subject, input, 5*time.Minute, accountID)
+	return utils.NATSRequest[ec2.Reservation](ctx, d.natsConn, subject, input, 5*time.Minute, accountID)
 }
 
 // TerminateWorkerInstances terminates nodegroup workers by routing a terminate
@@ -111,7 +114,7 @@ func (d *Daemon) terminateWorkerInstance(ctx context.Context, instanceID, accoun
 			break
 		}
 		if attempt < 2 {
-			time.Sleep(time.Duration(attempt+1) * time.Second)
+			terminateRetrySleep(time.Duration(attempt+1) * time.Second)
 		}
 	}
 	if errors.Is(err, nats.ErrNoResponders) {

spinifex/daemon/eks_worker_launch_test.go

@@ -17,10 +17,20 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+// noopTerminateRetrySleep replaces the terminate NoResponders backoff with a
+// no-op for the duration of the test, so retry paths do not burn real seconds.
+func noopTerminateRetrySleep(t *testing.T) {
+	t.Helper()
+	prev := terminateRetrySleep
+	terminateRetrySleep = func(time.Duration) {}
+	t.Cleanup(func() { terminateRetrySleep = prev })
+}
+
 // A worker with no owner (no responder on ec2.cmd.<id>) is already-gone, so
 // terminating it is an idempotent no-op success (a retried DeleteNodegroup /
 // scale-down must not wedge on drained instances).
 func TestTerminateWorkerInstances_NotFoundIsIdempotent(t *testing.T) {
+	noopTerminateRetrySleep(t)
 	nc, err := nats.Connect(sharedNATSURL)
 	require.NoError(t, err)
 	t.Cleanup(nc.Close)
@@ -105,6 +115,7 @@ func TestTerminateWorkerInstances_OwnerErrorSurfaces(t *testing.T) {
 // a stopped+wedged node's ENI pins the customer subnet/VPC undeletable and
 // DeleteCluster wedges in DELETING (mulga-siv-475).
 func TestTerminateWorkerInstances_StoppedFallbackToEC2Terminate(t *testing.T) {
+	noopTerminateRetrySleep(t)
 	nc, err := nats.Connect(sharedNATSURL)
 	require.NoError(t, err)
 	t.Cleanup(nc.Close)
@@ -133,6 +144,7 @@ func TestTerminateWorkerInstances_StoppedFallbackToEC2Terminate(t *testing.T) {
 // A NotFound payload from the ec2.terminate fallback (worker already drained) is
 // idempotent success, not a teardown failure.
 func TestTerminateWorkerInstances_StoppedFallbackNotFoundIdempotent(t *testing.T) {
+	noopTerminateRetrySleep(t)
 	nc, err := nats.Connect(sharedNATSURL)
 	require.NoError(t, err)
 	t.Cleanup(nc.Close)
@@ -150,6 +162,7 @@ func TestTerminateWorkerInstances_StoppedFallbackNotFoundIdempotent(t *testing.T
 // A non-NotFound error from the ec2.terminate fallback must surface so the
 // teardown backstop retries rather than orphaning the worker (and its ENI).
 func TestTerminateWorkerInstances_StoppedFallbackErrorSurfaces(t *testing.T) {
+	noopTerminateRetrySleep(t)
 	nc, err := nats.Connect(sharedNATSURL)
 	require.NoError(t, err)
 	t.Cleanup(nc.Close)

spinifex/daemon/gpu_pool_reconciler.go

@@ -0,0 +1,85 @@
+package daemon
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/mulgadc/spinifex/spinifex/vm"
+)
+
+// gpuPoolLiveStates are the instance states that legitimately still hold a
+// GPU pool entry: mid-launch, running, or mid-teardown (stopCleanup /
+// terminateCleanup release the slot before the instance settles into
+// Stopped/Terminated). Anything else — Stopped, Error, Terminated, or the
+// instance being entirely absent from the running map — means the slot
+// should already have been released and was not.
+var gpuPoolLiveStates = map[vm.InstanceState]bool{
+	vm.StateProvisioning: true,
+	vm.StatePending:      true,
+	vm.StateRunning:      true,
+	vm.StateStopping:     true,
+	vm.StateShuttingDown: true,
+}
+
+// gpuPoolReconciler frees GPU pool entries whose owning instance is absent
+// or has settled in a terminal, non-running state. It is the backstop for
+// the synchronous release in vm/crash_recovery.go: a node-down mid-cascade,
+// or any other gap that skips the synchronous release, would otherwise leak
+// the slot forever, since nothing else reclaims it. Node-local vm.Reaper run
+// by the GarbageCollector backstop, mirroring eniReconciler.
+type gpuPoolReconciler struct {
+	d *Daemon
+}
+
+var _ vm.Reaper = (*gpuPoolReconciler)(nil)
+
+// newGPUPoolReconciler builds the reconciler over the daemon. Returns nil
+// when the VM manager is absent so the caller can skip wiring it. The GPU
+// manager itself is read live from d.gpuManager on every sweep (not
+// captured here) since SIGHUP can replace it at runtime.
+func (d *Daemon) newGPUPoolReconciler() *gpuPoolReconciler {
+	if d.vmMgr == nil {
+		return nil
+	}
+	return &gpuPoolReconciler{d: d}
+}
+
+func (r *gpuPoolReconciler) Class() string         { return "gpu-pool" }
+func (r *gpuPoolReconciler) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
+
+// Sweep walks the GPU pool and releases any claimed entry whose instance is
+// absent from the running map or has settled in a non-running state.
+func (r *gpuPoolReconciler) Sweep(ctx context.Context) (int, error) {
+	mgr := r.d.gpuManager
+	if mgr == nil {
+		return 0, nil
+	}
+
+	reaped := 0
+	for _, entry := range mgr.Snapshot() {
+		if ctx.Err() != nil {
+			return reaped, ctx.Err()
+		}
+		if entry.InstanceID == "" || r.instanceHoldsSlot(entry.InstanceID) {
+			continue
+		}
+		slog.Warn("gpu-reconciler: releasing orphaned GPU pool entry",
+			"instanceId", entry.InstanceID, "pci", entry.Device.PCIAddress)
+		if err := mgr.Release(entry.InstanceID); err != nil {
+			slog.Warn("gpu-reconciler: release failed", "instanceId", entry.InstanceID, "err", err)
+			continue
+		}
+		reaped++
+	}
+	return reaped, nil
+}
+
+// instanceHoldsSlot reports whether instanceID names a VM in a state that
+// legitimately still holds its GPU pool entry.
+func (r *gpuPoolReconciler) instanceHoldsSlot(instanceID string) bool {
+	instance, ok := r.d.vmMgr.Get(instanceID)
+	if !ok {
+		return false
+	}
+	return gpuPoolLiveStates[r.d.vmMgr.Status(instance)]
+}

spinifex/daemon/gpu_pool_reconciler_test.go

@@ -0,0 +1,103 @@
+package daemon
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/gpu"
+	"github.com/mulgadc/spinifex/spinifex/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// claimMIGSliceForReconcilerTest seeds a one-slice MIG pool and claims it for
+// instanceID. Uses the MIG path (not whole-GPU) so the test never touches
+// real VFIO/sysfs state — MIG release is pure in-memory bookkeeping.
+func claimMIGSliceForReconcilerTest(t *testing.T, instanceID string) *gpu.Manager {
+	t.Helper()
+	mdevPath := filepath.Join(t.TempDir(), "mdev0")
+	require.NoError(t, os.WriteFile(mdevPath, []byte("x"), 0o600))
+
+	mgr := gpu.NewManager(nil)
+	mgr.AddMIGInstances(gpu.GPUDevice{PCIAddress: "0000:03:00.0"}, []gpu.MIGInstance{
+		{Profile: gpu.MIGProfile{Name: "1g.10gb"}, MdevPath: mdevPath},
+	})
+	_, _, err := mgr.Claim(instanceID, "1g.10gb")
+	require.NoError(t, err)
+	require.Equal(t, 0, mgr.Available(), "GPU slot must be claimed before the sweep")
+	return mgr
+}
+
+// TestGPUPoolReconciler_FreesOrphanedSlot is the regression test for
+// mulga-keptb: a GPU pool entry whose InstanceID names an instance absent
+// from the running map entirely must be released.
+func TestGPUPoolReconciler_FreesOrphanedSlot(t *testing.T) {
+	mgr := claimMIGSliceForReconcilerTest(t, "i-orphan")
+	d := &Daemon{vmMgr: vm.NewManager(), gpuManager: mgr}
+	r := d.newGPUPoolReconciler()
+	require.NotNil(t, r)
+
+	reaped, err := r.Sweep(context.Background())
+
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped)
+	assert.Equal(t, 1, mgr.Available(), "orphaned slot must return to the pool")
+}
+
+// TestGPUPoolReconciler_LeavesRunningInstanceAlone verifies the reconciler
+// does not touch a slot whose instance is actively running.
+func TestGPUPoolReconciler_LeavesRunningInstanceAlone(t *testing.T) {
+	mgr := claimMIGSliceForReconcilerTest(t, "i-running")
+	vmMgr := vm.NewManager()
+	vmMgr.Insert(&vm.VM{ID: "i-running", Status: vm.StateRunning})
+	d := &Daemon{vmMgr: vmMgr, gpuManager: mgr}
+	r := d.newGPUPoolReconciler()
+	require.NotNil(t, r)
+
+	reaped, err := r.Sweep(context.Background())
+
+	require.NoError(t, err)
+	assert.Equal(t, 0, reaped)
+	assert.Equal(t, 0, mgr.Available(), "a running instance's GPU slot must not be reclaimed")
+}
+
+// TestGPUPoolReconciler_FreesSlotForTerminalInstance covers the backstop case
+// for a stuck StateError instance still holding a GPU slot — the same
+// scenario the synchronous crash-recovery release targets directly, here
+// caught by the reconciler as a second line of defense.
+func TestGPUPoolReconciler_FreesSlotForTerminalInstance(t *testing.T) {
+	mgr := claimMIGSliceForReconcilerTest(t, "i-error")
+	vmMgr := vm.NewManager()
+	vmMgr.Insert(&vm.VM{ID: "i-error", Status: vm.StateError})
+	d := &Daemon{vmMgr: vmMgr, gpuManager: mgr}
+	r := d.newGPUPoolReconciler()
+	require.NotNil(t, r)
+
+	reaped, err := r.Sweep(context.Background())
+
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped)
+	assert.Equal(t, 1, mgr.Available())
+}
+
+// TestGPUPoolReconciler_NoGPUManager_NoOp verifies the reconciler is a
+// harmless no-op when GPU passthrough is disabled.
+func TestGPUPoolReconciler_NoGPUManager_NoOp(t *testing.T) {
+	d := &Daemon{vmMgr: vm.NewManager()}
+	r := d.newGPUPoolReconciler()
+	require.NotNil(t, r)
+
+	reaped, err := r.Sweep(context.Background())
+
+	require.NoError(t, err)
+	assert.Equal(t, 0, reaped)
+}
+
+// TestNewGPUPoolReconciler_NilWithoutVMManager verifies the constructor
+// refuses to build a reconciler with no VM manager to query.
+func TestNewGPUPoolReconciler_NilWithoutVMManager(t *testing.T) {
+	d := &Daemon{}
+	assert.Nil(t, d.newGPUPoolReconciler())
+}