fix: admin init --force preserves keys, credentials and CA (#536 )

ci: publish eks-node-gpu + ecs-agent-gpu AMIs
Expand the publish workflow to build the two Ubuntu NVIDIA GPU variants alongside the Alpine eks-node/ecs-agent AMIs. The image build+publish scripts are already manifest-driven and Ubuntu-aware; only the workflow lacked the GPU legs. Refactor the single-job body into a matrix over the four image manifests, resolved by a new 'variant' input (all|cpu|gpu) so the slow 16G GPU legs can be skipped when only the CPU AMIs need republishing. GPU images do not fit a RAM-backed /tmp, so set SYSTEM_IMAGE_BUILD_DIR onto the runner's on-disk _temp; legs run serially (max-parallel 1) on the single ci runner.
2026-10-24 05:04:03 +00:00 · 2026-07-14 14:46:12 +10:00 · 2026-07-14 14:05:58 +10:00 · 2026-07-14 13:43:02 +10:00 · 2026-07-14 13:37:14 +10:00 · 2026-07-14 12:43:42 +10:00
1273 changed files with 71350 additions and 16702 deletions
@@ -1,7 +1,6 @@
 // Package main implements the e2e-analyze GitHub Action.
 //
-// Stage 1 of docs/development/improvements/e2e-go-failure-analysis.md:
-// cluster failures from go-junit-report's junit-*.xml by error signature,
+// Stage 1: cluster failures from go-junit-report's junit-*.xml by error signature,
 // pick the earliest non-cascade failure per suite as the likely root cause,
 // and render a human-readable report.
 //
@@ -463,8 +462,7 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 	return rep, nil
 }

-// Render writes the markdown report. Format is documented in
-// docs/development/improvements/e2e-go-failure-analysis.md (Stage 1).
+// Render writes the markdown report (Stage 1).
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)
@@ -1,7 +1,6 @@
 package main

-// Stage 2 of docs/development/improvements/e2e-go-failure-analysis.md:
-// for each failure surfaced by Stage 1, materialise a per-failure bundle
+// Stage 2: for each failure surfaced by Stage 1, materialise a per-failure bundle
 // directory containing the failure summary, a time-window slice of the
 // daemon journal, a testcase-window slice of the suite's stdout log, and
 // any per-test artifacts the harness already dumped at run-time.
@@ -1,7 +1,6 @@
 package main

-// Stage 2 of docs/development/improvements/e2e-go-failure-analysis.md:
-// per-failure log slicers used by bundle.go to materialise time-window /
+// Stage 2: per-failure log slicers used by bundle.go to materialise time-window /
 // testcase-window log excerpts next to each named failure.

 import (
@@ -16,6 +16,10 @@ permissions:

 env:
  GIT_BRANCH: ${{ github.ref_name }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
  ddil:
@@ -200,6 +204,16 @@ jobs:
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
          ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-ddil-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 30
@@ -17,6 +17,10 @@ permissions:
 env:
  GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
  matrix:
@@ -35,6 +39,7 @@ jobs:
          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
          - { cell: 17, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
          - { cell: 18, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-single, extra: "reboot-single" }
+          - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }

    runs-on: [ self-hosted, "${{ matrix.runner }}" ]
    timeout-minutes: 35
@@ -121,7 +126,12 @@ jobs:
        env:
          MATRIX_NET: ${{ matrix.net }}
        run: |
-          if [ "$MATRIX_NET" = "dhcp" ]; then
+          if [ "$MATRIX_NET" = "nat" ]; then
+            # Routed-NAT mode with a static public pool: exercises both the
+            # Tier-1 transit datapath and the Tier-2 EIP ingress lane.
+            export SPX_EXTERNAL_MODE=nat
+            export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
+          elif [ "$MATRIX_NET" = "dhcp" ]; then
            unset SPX_EXTERNAL_POOL
            export SPX_EXTERNAL_SOURCE=dhcp
          else
@@ -154,6 +164,28 @@ jobs:
               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
             done; exit \$FAIL"

+      - name: Run NAT Uplink E2E
+        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'nat-single'
+        working-directory: mulga/scripts/tofu-cluster
+        # Skips cleanly (exit 0) when the tested branch predates the natuplink
+        # suite so scheduled runs against dev stay green until the merge lands.
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
+             if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
+             echo '::group::suite natuplink'; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./natuplink.test -test.v -test.timeout=30m \
+               2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
+             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
+             if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
+
      - name: Run Multi Node E2E
        if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
        working-directory: mulga/scripts/tofu-cluster
@@ -312,9 +344,19 @@ jobs:
            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
          fi
          tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
+          timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
  # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
  # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
  # matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
@@ -44,6 +44,10 @@ permissions:
 env:
  GIT_BRANCH: ${{ github.ref_name }}
  TEST_RUN: ${{ inputs.test_run || '' }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
  # ---------------------------------------------------------------------------
@@ -263,6 +267,15 @@ jobs:
            "$VM":/tmp/start-net-capture.sh
          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
  single_suite:
    name: Single env · suite
    needs: single_provision
@@ -458,10 +471,20 @@ jobs:
          reap TERM; sleep 2; reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-td-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
  # ---------------------------------------------------------------------------
  # Multinode env
  # ---------------------------------------------------------------------------
@@ -612,6 +635,15 @@ jobs:
            ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true
          done

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
  multi_suite:
    name: Multi env · suite
    needs: multi_provision
@@ -779,10 +811,20 @@ jobs:
          reap TERM; sleep 2; reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-td-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
  e2e_suites_status:
    name: E2E suite blocks · summary
    if: always()
@@ -31,6 +31,10 @@ env:
  GIT_BRANCH: ${{ github.ref_name }}
  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam' }}
  TEST_RUN: ${{ inputs.test_run || '' }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

 jobs:
  # Shadow-mode suite selector. Computes which suites the diff *would* run and logs it to the job summary.
@@ -457,10 +461,20 @@ jobs:
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-single-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
      - name: Fail job if any suite failed
        if: always()
        run: |
@@ -871,10 +885,20 @@ jobs:
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-multi-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14
+
      - name: Fail job if any suite failed
        if: always()
        run: |
@@ -0,0 +1,369 @@
+# GPU E2E — drives wattle's persistent, shared-host Spinifex stack over SSH to
+# exercise GPU passthrough (raw EC2, ECS task GPU exposure, EKS pod GPU
+# exposure). See docs/development/feature/wattle-deploy-and-ci.md (Part B).
+#
+# ---------------------------------------------------------------------------
+# ONE-TIME HOST-SIDE SETUP — NOT part of this repo, a human must do this:
+#
+#   1. Register ONE self-hosted GitHub Actions runner on a banksia host with
+#      the dedicated label `ci-gpu` (in addition to `self-hosted`). Do
+#      NOT add it to the `ci-single` / `ci-multi` pool and do NOT give any
+#      other runner the `ci-gpu` label — a second labelled runner would
+#      let two GPU jobs race the single RTX A1000. With exactly one runner
+#      carrying the label, `runs-on: [self-hosted, ci-gpu]` alone forces
+#      serialization; the `concurrency:` block below queues same-group runs
+#      when that runner is already busy with a live job.
+#   2. Drop the wattle SSH private key on that runner's filesystem and wire
+#      it into this repo as `secrets.WATTLE_SSH_KEY` (raw private key
+#      contents). Optionally override `vars.WATTLE_SSH_HOST` /
+#      `vars.WATTLE_SSH_USER` if they differ from the defaults below.
+#   3. Confirm the runner user can drive wattle's control plane over SSH. The
+#      job calls `spx` (read-only checks) and the `aws` CLI with the already-
+#      configured `spinifex` profile directly — no wrapper scripts, no install
+#      on wattle (tf-user is already in the `spinifex` group there):
+#        ssh tf-user@192.168.1.13 'AWS_PROFILE=spinifex aws ec2 describe-regions'
+#   4. Manual-work mutex: an agent doing hands-on GPU testing on wattle can
+#      block CI by touching a sentinel file in the RUNNER's home directory
+#      (NOT on wattle):
+#        touch ~/.ci-gpu-paused     # blocks CI
+#        rm ~/.ci-gpu-paused        # resumes CI
+#      or by setting the repo/environment variable `CI_GPU_PAUSED=true`
+#      (no runner filesystem access needed). Either one causes this workflow
+#      to skip cleanly (job succeeds, all work steps skipped) instead of
+#      queuing behind manual work indefinitely.
+#
+#   Runbook: docs/development/feature/wattle-deploy-and-ci.md (Part B).
+# ---------------------------------------------------------------------------
+
+name: GPU E2E
+
+on:
+  workflow_dispatch:
+    inputs:
+      test_run:
+        description: "Optional -test.run regex scoped to tests/e2e/gpu (empty = run all GPU tests)"
+        required: false
+        default: ""
+  push:
+    branches:
+      - main
+      - dev
+    paths: &gpu_paths
+      - "spinifex/gpu/**"
+      - "spinifex/instancetypes/**"
+      - "spinifex/daemon/gpu_adapter.go"
+      - "spinifex/daemon/gpu_pool_reconciler.go"
+      - "spinifex/handlers/ecs/gpu_report.go"
+      - "spinifex/handlers/eks/nodegroup.go"
+      - "spinifex/handlers/eks/nodegroup_userdata.go"
+      - "spinifex/vm/vm.go"
+      - "spinifex/vm/lifecycle.go"
+      - "cmd/ecs-agent/gpu_discovery.go"
+      - "cmd/ecs-agent/gpu_ledger.go"
+      - "cmd/spinifex/cmd/admin_gpu.go"
+      - "tests/e2e/gpu/**"
+      - "scripts/images/eks-node-gpu/**"
+      - "scripts/images/ecs-agent-gpu/**"
+      - ".github/workflows/gpu-e2e.yml"
+  pull_request:
+    branches:
+      - main
+      - dev
+    paths: *gpu_paths
+
+permissions:
+  contents: read
+
+# Single dedicated runner (see setup note 1 above) + this group together
+# enforce single-flight on the one RTX A1000: a second dispatch has nowhere
+# else to land (unique label) and, if it somehow overlaps in scheduling,
+# queues here rather than cancelling/stomping the in-flight run.
+concurrency:
+  group: ci-gpu
+  cancel-in-progress: false
+
+env:
+  # OTLP export to the central Elastic sink (wattle is the sink itself — see
+  # Part A.2). Repo variable overrides; carried for correlation even though
+  # the GPU suite's own daemon-side spans are emitted by wattle's already-
+  # configured obs-agent, not by this job.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
+  WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
+  WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
+  TEST_RUN: ${{ inputs.test_run || '' }}
+
+jobs:
+  gpu_e2e:
+    name: GPU E2E (wattle)
+    runs-on: [self-hosted, ci-gpu]
+    timeout-minutes: 75
+    env:
+      ARTIFACT_DIR: /tmp/spinifex-gpu-e2e-artifacts-${{ github.run_id }}
+    steps:
+      # Manual-work mutex gate — see setup note 4 above. Checked BEFORE any
+      # SSH to wattle so a paused run touches the box zero times.
+      - name: Manual-work mutex gate
+        id: gate
+        env:
+          REPO_VAR_PAUSED: ${{ vars.CI_GPU_PAUSED }}
+        run: |
+          set -euo pipefail
+          PAUSED=false
+          if [ -f "$HOME/.ci-gpu-paused" ]; then
+            PAUSED=true
+            echo "::warning::ci-gpu runner paused via ~/.ci-gpu-paused sentinel on the runner — skipping this run (manual GPU work in progress on wattle). Remove the file to resume CI."
+          fi
+          if [ "$(echo "${REPO_VAR_PAUSED:-}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
+            PAUSED=true
+            echo "::warning::ci-gpu paused via repo/environment variable CI_GPU_PAUSED=true — skipping this run."
+          fi
+          echo "paused=${PAUSED}" >> "$GITHUB_OUTPUT"
+
+      - name: Paused — nothing to do
+        if: steps.gate.outputs.paused == 'true'
+        run: echo "ci-gpu is paused; job exits cleanly without touching wattle. See gate step above for the reason."
+
+      - name: Clean Workspace
+        if: steps.gate.outputs.paused != 'true'
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/checkout@v7
+        if: steps.gate.outputs.paused != 'true'
+
+      - uses: actions/setup-go@v6
+        if: steps.gate.outputs.paused != 'true'
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Install wattle SSH key
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          install -m 0600 /dev/null "${RUNNER_TEMP}/wattle-ci"
+          echo "${{ secrets.WATTLE_SSH_KEY }}" > "${RUNNER_TEMP}/wattle-ci"
+          echo "ssh_opts=-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${RUNNER_TEMP}/wattle-ci" >> "$GITHUB_ENV"
+
+      # Healthcheck: confirm wattle's persistent AWS-compat control plane is
+      # actually up before spending time building/shipping the test binary.
+      - name: Healthcheck — describe-regions
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" \
+            'AWS_PROFILE=spinifex aws ec2 describe-regions --output text'
+
+      # Ensure the base EC2 GPU AMI is present — the raw-EC2 GPU passthrough
+      # test is the always-on baseline, so CI self-provisions it rather than
+      # depending on a manual import. `--name` selects the catalog entry (pulls
+      # the prebuilt image from iso.mulgadc.com); `--ami-name` is REQUIRED to
+      # name the AMI ubuntu-26.04-nvidia-gpu-x86_64 — without it the import
+      # falls back to the generic ami-<distro>-<version>-<arch>, which the
+      # tests/e2e/gpu filter would never match. Idempotent — skipped once
+      # wattle's persistent loop-fs already has the named AMI (the 4.4G source
+      # qcow2 is cached in the images dir, so a re-import wouldn't re-download).
+      # `spx admin images import` writes local spinifex storage, so it needs
+      # the `spinifex` group active (sg); aws-CLI calls hit the gateway over
+      # HTTPS and don't.
+      - name: Ensure base GPU AMI present (import-if-absent)
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          REMOTE='
+            set -e
+            BASE=ubuntu-26.04-nvidia-gpu-x86_64
+            present() {
+              AWS_PROFILE=spinifex aws ec2 describe-images \
+                --filters "Name=name,Values=$BASE" "Name=state,Values=available" \
+                --query "Images[0].ImageId" --output text 2>/dev/null
+            }
+            ID=$(present)
+            if [ -n "$ID" ] && [ "$ID" != "None" ]; then
+              echo "base GPU AMI already present: $BASE = $ID"; exit 0
+            fi
+            echo "base GPU AMI $BASE absent — importing prebuilt from catalog..."
+            sg spinifex -c "AWS_PROFILE=spinifex spx admin images import --name $BASE --ami-name $BASE"
+            for i in 1 2 3 4 5 6; do
+              ID=$(present)
+              [ -n "$ID" ] && [ "$ID" != "None" ] && break
+              sleep 5
+            done
+            [ -n "$ID" ] && [ "$ID" != "None" ] || { echo "import ran but $BASE still not available" >&2; exit 1; }
+            echo "imported base GPU AMI: $BASE = $ID"
+          '
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
+
+      # Preflight — the base GPU AMI is REQUIRED: the setup step above just
+      # ensured it, so a miss here means the import silently failed. Hard-fail
+      # rather than run a false-green all-skipped suite. ECS/EKS node AMIs are
+      # opt-in tiers (heavier — node AMIs + cluster spin-up); their tests
+      # self-skip when absent, so only warn.
+      - name: Preflight — verify GPU AMIs
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          REMOTE='
+            set -e
+            present() {
+              AWS_PROFILE=spinifex aws ec2 describe-images \
+                --filters "Name=name,Values=$1" "Name=state,Values=available" \
+                --query "Images[0].ImageId" --output text 2>/dev/null
+            }
+            BASE=ubuntu-26.04-nvidia-gpu-x86_64
+            ID=$(present "$BASE")
+            if [ -z "$ID" ] || [ "$ID" = "None" ]; then
+              echo "required base GPU AMI $BASE missing — cannot run the baseline EC2 GPU test" >&2
+              exit 1
+            fi
+            echo "required: $BASE = $ID"
+            for OPT in spinifex-ecs-node-gpu spinifex-eks-node spinifex-eks-node-gpu; do
+              OID=$(present "$OPT")
+              if [ -z "$OID" ] || [ "$OID" = "None" ]; then
+                echo "::warning::$OPT not imported — its GPU test(s) will skip (opt-in tier)"
+              else
+                echo "opt-in: $OPT = $OID"
+              fi
+            done
+          '
+          ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
+
+      # Same build command CI's other e2e workflows use to cross-compile a
+      # suite into a shippable binary (tests/e2e/Makefile e2e-build target,
+      # SUITES_BUILD scoped to just gpu). Building on the runner keeps go
+      # toolchain / module cache off wattle entirely.
+      - name: Build GPU e2e test binary on runner
+        if: steps.gate.outputs.paused != 'true'
+        working-directory: tests/e2e
+        run: |
+          set -euo pipefail
+          make e2e-build SUITES_BUILD=gpu
+          ls -la _bin/gpu.test
+
+      - name: Install go-junit-report on runner
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          go install github.com/jstemmer/go-junit-report/v2@latest
+          echo "$HOME/go/bin" >> "$GITHUB_PATH"
+
+      - name: Ship GPU e2e test binary to wattle
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set -euo pipefail
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          ssh ${{ env.ssh_opts }} "$VM" "mkdir -p \$HOME/spinifex-gpu-e2e-bin"
+          scp ${{ env.ssh_opts }} tests/e2e/_bin/gpu.test "$VM":spinifex-gpu-e2e-bin/gpu.test
+
+      # Same invocation shape e2e.yml uses to run a pre-built suite binary
+      # (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
+      # -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
+      # net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
+      # the advertised gateway (mulga-7suds), so the override is required
+      # here (proper fix tracked separately: tests/e2e/harness/env.go should
+      # prefer spinifex.toml's advertised bridge).
+      - name: Run GPU e2e suite on wattle
+        id: e2e
+        if: steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          if [ -n "${TEST_RUN}" ]; then
+            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
+          else
+            TEST_RUN_FLAG=""
+          fi
+          ssh -tt ${{ env.ssh_opts }} "$VM" \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-gpu-e2e-bin; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               SPINIFEX_WAN_IP='${WATTLE_SSH_HOST}' \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./gpu.test -test.v -test.timeout=60m ${TEST_RUN_FLAG} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-gpu.log'; \
+             exit \${PIPESTATUS[0]}"
+          E2E_RC=$?
+          echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
+          exit 0
+
+      - name: Collect artifacts from wattle
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          ssh ${{ env.ssh_opts }} "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
+            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
+          echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
+
+      - name: Convert test log to JUnit XML
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          shopt -s nullglob
+          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
+            BASE=$(basename "$LOG" .log)
+            SUITE=${BASE#test-}
+            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
+          done
+
+      - name: Publish test summary
+        if: always() && steps.gate.outputs.paused != 'true'
+        uses: ./.github/actions/junit-report
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          title: "GPU E2E results"
+
+      - name: Analyze e2e failures
+        if: always() && steps.gate.outputs.paused != 'true'
+        uses: ./.github/actions/e2e-analyze
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          log-dir: ${{ env.ARTIFACT_DIR }}
+          title: "GPU E2E failure analysis"
+
+      - uses: actions/upload-artifact@v7
+        if: always() && steps.gate.outputs.paused != 'true'
+        with:
+          name: gpu-e2e-artifacts-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      # Cleanup is NEVER a host teardown: it never runs setup-ovn.sh,
+      # dev-reset, or cluster-reset. It only reconciles EC2-API-level state —
+      # terminate any GPU-typed instance still pending/running (the Go
+      # suite's own t.Cleanup already does this on normal pass/fail exit;
+      # this is the backstop for a hard `-test.timeout` kill or a workflow
+      # cancellation, which skip deferred Go cleanups). Termination triggers
+      # gpuManager.Release() on wattle, returning the A1000 to the pool for
+      # the next queued run or manual test.
+      - name: Cleanup — terminate leftover GPU instances, release GPU
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          set +e
+          VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
+          REMOTE='
+            set +e
+            GPU_TYPES=$(AWS_PROFILE=spinifex aws ec2 describe-instance-types \
+              --query "InstanceTypes[?GpuInfo!=null].InstanceType" --output text | tr "\t" ",")
+            if [ -z "$GPU_TYPES" ]; then
+              echo "no GPU instance types advertised; nothing to reconcile"
+              exit 0
+            fi
+            IDS=$(AWS_PROFILE=spinifex aws ec2 describe-instances \
+              --filters "Name=instance-type,Values=${GPU_TYPES}" "Name=instance-state-name,Values=pending,running" \
+              --query "Reservations[].Instances[].InstanceId" --output text)
+            if [ -z "$IDS" ] || [ "$IDS" = "None" ]; then
+              echo "no leftover GPU instances — GPU already released"
+              exit 0
+            fi
+            echo "terminating leftover GPU instance(s): $IDS"
+            AWS_PROFILE=spinifex aws ec2 terminate-instances --instance-ids $IDS
+          '
+          ssh ${{ env.ssh_opts }} "$VM" "$REMOTE"
+
+      - name: Fail job if GPU suite failed
+        if: always() && steps.gate.outputs.paused != 'true'
+        run: |
+          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
+            echo "::error::GPU e2e suite failed (rc=${{ steps.e2e.outputs.rc }}) — see analysis above; raw ES/APM traces at ${SPINIFEX_OTLP_ENDPOINT} (ci.run_id=${{ github.run_id }})"
+            exit 1
+          fi
@@ -1,16 +1,34 @@
-name: Publish EKS node AMI
+name: Publish EKS + ECS node AMIs

-# Builds the eks-node system AMI (libguestfs, on the runner) and publishes the
+# Builds the system AMIs (libguestfs, on the runner) and publishes each
 # compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
-# iso.mulgadc.com/system-ami/. Operators then install it on any cluster with:
+# iso.mulgadc.com/system-ami/. Four images ship:
+#   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
+#   spinifex-ecs-node       Alpine ECS container instance
+#   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
+#   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
+# Operators then install them on any cluster with:
 #   spx admin images import --name spinifex-eks-node
+#   spx admin images import --name spinifex-ecs-node
+#   spx admin images import --name spinifex-eks-node-gpu
+#   spx admin images import --name spinifex-ecs-node-gpu
 #
 # Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
-# action, not something every push should trigger.
+# action, not something every push should trigger. The `variant` input scopes
+# which images build — the Ubuntu GPU legs are 16G and much slower (NVIDIA
+# driver download + DKMS build), so publishing only the CPU AMIs stays cheap.

 on:
  workflow_dispatch:
    inputs:
+      variant:
+        description: "Which images to build"
+        type: choice
+        default: all
+        options:
+          - all
+          - cpu
+          - gpu
      dry_run:
        description: "Build + convert only; skip the R2 upload"
        type: boolean
@@ -24,9 +42,37 @@ concurrency:
  cancel-in-progress: false

 jobs:
-  publish:
-    name: Build + publish eks-node AMI
+  matrix:
+    name: Resolve image matrix
    runs-on: [ self-hosted, ci-single ]
+    outputs:
+      images: ${{ steps.gen.outputs.images }}
+    steps:
+      - name: Select images for variant
+        id: gen
+        env:
+          VARIANT: ${{ inputs.variant }}
+        run: |
+          set -euo pipefail
+          cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
+          gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
+          case "${VARIANT}" in
+            cpu) items="${cpu}" ;;
+            gpu) items="${gpu}" ;;
+            *)   items="${cpu},${gpu}" ;;
+          esac
+          echo "images={\"include\":[${items}]}" >> "$GITHUB_OUTPUT"
+
+  publish:
+    name: Build + publish ${{ matrix.name }}
+    needs: matrix
+    runs-on: [ self-hosted, ci-single ]
+    strategy:
+      fail-fast: false
+      # ci-single is one runner and each GPU build needs ~16G of scratch disk;
+      # serialise the legs so they never contend for /_temp or the mirror.
+      max-parallel: 1
+      matrix: ${{ fromJSON(needs.matrix.outputs.images) }}

    steps:
      - name: Checkout Spinifex
@@ -73,7 +119,7 @@ jobs:
            rm -rf /tmp/awscliv2.zip /tmp/aws-install
          fi

-      - name: Build + publish to R2
+      - name: Build + publish ${{ matrix.name }} to R2
        working-directory: spinifex
        env:
          # Map the repo's R2 secrets onto the names publish-system-image.sh reads.
@@ -82,15 +128,20 @@ jobs:
          R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
          R2_BUCKET: spinifex-iso
          DRY_RUN: ${{ inputs.dry_run }}
+          MANIFEST: ${{ matrix.manifest }}
+          # Ubuntu GPU images are 16G — a RAM-backed /tmp cannot hold the qcow2 +
+          # expanded raw, so point the build scratch at the runner's on-disk
+          # _temp. Harmless for the small Alpine legs too.
+          SYSTEM_IMAGE_BUILD_DIR: ${{ runner.temp }}/system-image-build
        run: |
          set -euo pipefail
          # libguestfs needs to read a host kernel; Ubuntu ships it mode 0600.
          # direct backend = no libvirt on the runner.
          export LIBGUESTFS_BACKEND=direct
          sudo chmod 0644 /boot/vmlinuz-* || true
+          mkdir -p "${SYSTEM_IMAGE_BUILD_DIR}"
          FLAGS=(--build)
          if [[ "${DRY_RUN}" == "true" ]]; then
            FLAGS+=(--dry-run)
          fi
-          ./scripts/publish-system-image.sh \
-            scripts/images/eks-node/manifest.conf "${FLAGS[@]}"
+          ./scripts/publish-system-image.sh "${MANIFEST}" "${FLAGS[@]}"
@@ -34,7 +34,7 @@ jobs:
      - name: Set up Go
        uses: actions/setup-go@v6
        with:
-          go-version: "1.26.4"
+          go-version: "1.26.5"

      - name: Build microVM artifacts (lb-agent + kernel + initramfs)
        run: cd spinifex && make build-microvm-image
@@ -218,9 +218,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base

 install-go:
-	@echo -e "\n....Installing Go 1.26.4 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.5 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.4.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.5.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -192,6 +192,10 @@ While Spinifex is architected and implemented by experienced engineers, we lever

 This hybrid approach ensures Spinifex benefits from both proven engineering expertise and modern development acceleration, while maintaining the quality and reliability standards required for production infrastructure.

+## Trademarks
+
+"AWS", "Amazon Web Services", and all related service names (EC2, EBS, S3, VPC, IAM, and others) are trademarks of Amazon.com, Inc. or its affiliates. Mulga Defense Corporation and Spinifex are independent and not affiliated with, endorsed by, or sponsored by Amazon.com, Inc. or Amazon Web Services, Inc. References to AWS services describe interoperability and compatibility only.
+
 ## License

 Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
@@ -5,6 +5,7 @@ After=spinifex-nats.service spinifex-daemon.service
 Wants=spinifex-nats.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-gw
@@ -6,6 +6,7 @@ After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.servi
 Wants=openvswitch-ipsec.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-daemon
@@ -4,6 +4,7 @@ PartOf=spinifex.target
 Before=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-nats
@@ -5,6 +5,10 @@ After=spinifex-nats.service
 Wants=spinifex-nats.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+# Guest block I/O arrives with no caller trace; sample those roots at 5% so
+# they don't drown request traces. Request-linked spans are always kept.
+Environment=MULGA_ROOT_TRACE_RATIO=0.05
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-storage
@@ -0,0 +1,56 @@
+[Unit]
+Description=Spinifex QMP Telemetry Collector (guest metrics)
+PartOf=spinifex.target
+After=spinifex-nats.service spinifex-daemon.service
+Wants=spinifex-nats.service
+
+[Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+Type=simple
+Slice=spinifex-system.slice
+# spinifex-daemon user: the collector reads the daemon's telemetry QMP
+# sockets in /run/spinifex and the daemon-owned QEMU /proc entries.
+User=spinifex-daemon
+Group=spinifex
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
+ExecStart=/usr/local/bin/spx service qmp-collector start
+Restart=on-failure
+RestartSec=5
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
+Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
+# Same runtime dir as the daemon: telemetry sockets + metadata live there
+Environment=XDG_RUNTIME_DIR=/run/spinifex
+
+# RG-9: read-mostly telemetry tier — no capabilities, no device access
+ProtectSystem=full
+PrivateDevices=yes
+NoNewPrivileges=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem
+ReadWritePaths=/var/lib/spinifex/spinifex /run/spinifex
+
+# RG-9: hardening baseline
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+# ProtectProc must stay default: hidepid=invisible hides the QEMU vCPU threads
+# (dumpable=0) this collector must read from /proc/<tid>/stat, even same-uid.
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=512
+
+[Install]
+WantedBy=spinifex.target
@@ -4,6 +4,7 @@ PartOf=spinifex.target
 After=spinifex-awsgw.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-ui
@@ -5,6 +5,7 @@ After=spinifex-nats.service spinifex-predastore.service
 Wants=spinifex-nats.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-viperblock
@@ -5,6 +5,7 @@ After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ope
 Wants=spinifex-nats.service ovn-controller.service

 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-vpcd
@@ -2,7 +2,7 @@
 Description=Spinifex Infrastructure Platform
 After=network-online.target
 Wants=network-online.target
-Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
+Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service

 [Install]
 WantedBy=multi-user.target
@@ -43,6 +43,11 @@ type Agent struct {
 	// cred serves task IAM role credentials at 169.254.170.2 (nil in unit tests).
 	cred *credEndpoint

+	// gpu is the local pinning ledger over the host's discovered GPU UUIDs.
+	// Always non-nil (newAgent seeds an empty one); New() reseeds it from
+	// nvidia-smi discovery.
+	gpu *gpuLedger
+
 	closers []func() error
 }

@@ -61,6 +66,7 @@ func newAgent(cfg config, id identity, cp controlPlane, puller ctrruntime.ImageP
 		reg:      newRegistrar(cp, id),
 		hb:       newHeartbeater(cp, id, cfg.Heartbeat),
 		netns:    newTaskNetns(execNetRunner{}),
+		gpu:      newGPULedger(id.Capacity.GPUIDs),
 	}
 }

@@ -84,7 +90,7 @@ func New(cfg config) (*Agent, error) {
 		InstanceID:   meta.InstanceID,
 		AZ:           meta.AZ,
 		Hostname:     host,
-		Capacity:     detectCapacity(),
+		Capacity:     detectCapacity(discoverNvidiaGPUs(execCommandRunner)),
 		AgentVersion: version,
 	}

@@ -153,11 +159,15 @@ func (a *Agent) Stop() error {
 	return firstErr
 }

-// detectCapacity reports the host's total CPU units (1 vCPU = 1024) and memory.
-func detectCapacity() bus.InstanceCapacity {
+// detectCapacity reports the host's total CPU units (1 vCPU = 1024), memory,
+// and GPU count/UUIDs from the caller's nvidia-smi discovery (nil on a
+// non-GPU host).
+func detectCapacity(gpuUUIDs []string) bus.InstanceCapacity {
 	return bus.InstanceCapacity{
 		CPU:       goruntime.NumCPU() * 1024,
 		MemoryMiB: memTotalMiB(),
+		GPU:       len(gpuUUIDs),
+		GPUIDs:    gpuUUIDs,
 	}
 }

@@ -42,7 +42,7 @@ func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
 }

 func TestDetectCapacity_Positive(t *testing.T) {
-	c := detectCapacity()
+	c := detectCapacity(nil)
 	if c.CPU <= 0 {
 		t.Errorf("CPU = %d, want > 0", c.CPU)
 	}
@@ -50,4 +50,17 @@ func TestDetectCapacity_Positive(t *testing.T) {
 	if c.MemoryMiB < 0 {
 		t.Errorf("MemoryMiB = %d, want >= 0", c.MemoryMiB)
 	}
+	if c.GPU != 0 || c.GPUIDs != nil {
+		t.Errorf("no GPU UUIDs given: want GPU=0/GPUIDs=nil, got %d/%v", c.GPU, c.GPUIDs)
+	}
+}
+
+func TestDetectCapacity_WithGPUUUIDs(t *testing.T) {
+	c := detectCapacity([]string{"GPU-aaa", "GPU-bbb"})
+	if c.GPU != 2 {
+		t.Errorf("GPU = %d, want 2", c.GPU)
+	}
+	if len(c.GPUIDs) != 2 {
+		t.Errorf("GPUIDs = %v, want 2 entries", c.GPUIDs)
+	}
 }
@@ -25,9 +25,13 @@ type controlPlane interface {
 	Register(id identity) error
 	// SubmitTaskState reports a task's RUNNING/STOPPED transition.
 	SubmitTaskState(st bus.TaskState) error
-	// PollAssignments drains this instance's assignment inbox, acking the taskIDs
-	// accepted on the previous poll. Returns the still-pending assignments.
-	PollAssignments(cluster, instance string, ack []string) ([]bus.Assign, error)
+	// PollAssignments drains this instance's assignment + stop inboxes, acking the
+	// assign taskIDs and stop taskIDs accepted on the previous poll. Returns the
+	// still-pending assignments and stop directives.
+	PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error)
+	// ReportTaskGPU reports the local ledger's pinned device UUIDs for a task's
+	// containers, so the control plane can populate DescribeTasks gpuIds.
+	ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error
 }

 // gatewayControlPlane implements controlPlane over the SigV4 ECS gateway client.
@@ -73,6 +77,11 @@ func (g *gatewayControlPlane) Register(id identity) error {
 		},
 		VersionInfo: &ecs.VersionInfo{AgentVersion: aws.String(id.AgentVersion)},
 	}
+	if len(id.Capacity.GPUIDs) > 0 {
+		in.TotalResources = append(in.TotalResources, &ecs.Resource{
+			Name: aws.String("GPU"), Type: aws.String("STRINGSET"), StringSetValue: aws.StringSlice(id.Capacity.GPUIDs),
+		})
+	}
 	body, err := json.Marshal(in)
 	if err != nil {
 		return fmt.Errorf("marshal register: %w", err)
@@ -111,19 +120,36 @@ func (g *gatewayControlPlane) SubmitTaskState(st bus.TaskState) error {
 	return nil
 }

-func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ack []string) ([]bus.Assign, error) {
-	in := &handlers_ecs.PollAssignmentsInput{Cluster: cluster, ContainerInstance: instance, AckTaskIDs: ack}
+func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
+	in := &handlers_ecs.PollAssignmentsInput{
+		Cluster:           cluster,
+		ContainerInstance: instance,
+		AckTaskIDs:        ackAssigns,
+		AckStopIDs:        ackStops,
+	}
 	body, err := json.Marshal(in)
 	if err != nil {
-		return nil, fmt.Errorf("marshal poll: %w", err)
+		return nil, nil, fmt.Errorf("marshal poll: %w", err)
 	}
 	resp, err := g.client.Call("PollAssignments", body)
 	if err != nil {
-		return nil, fmt.Errorf("poll: %w", err)
+		return nil, nil, fmt.Errorf("poll: %w", err)
 	}
 	var out handlers_ecs.PollAssignmentsOutput
 	if err := json.Unmarshal(resp, &out); err != nil {
-		return nil, fmt.Errorf("decode poll: %w", err)
+		return nil, nil, fmt.Errorf("decode poll: %w", err)
 	}
-	return out.Assignments, nil
+	return out.Assignments, out.Stops, nil
+}
+
+func (g *gatewayControlPlane) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
+	in := &handlers_ecs.ReportTaskGPUInput{Cluster: cluster, Task: task, Containers: containers}
+	body, err := json.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("marshal report-task-gpu: %w", err)
+	}
+	if _, err := g.client.Call("ReportTaskGPU", body); err != nil {
+		return fmt.Errorf("report task gpu: %w", err)
+	}
+	return nil
 }
@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
+)
+
+// fixedCredsProvider is a static credentials.CredentialsProvider for tests
+// that only need a valid SigV4 signature, not real IMDS rotation.
+type fixedCredsProvider struct{}
+
+func (fixedCredsProvider) Retrieve(context.Context) (credentials.Credentials, error) {
+	return credentials.Credentials{AccessKeyID: "AKIATEST", SecretAccessKey: "secret"}, nil
+}
+
+func newTestGatewayControlPlane(t *testing.T, srv *httptest.Server) *gatewayControlPlane {
+	t.Helper()
+	cp, err := newGatewayControlPlane(config{GatewayURL: srv.URL, Region: "us-east-1"}, fixedCredsProvider{})
+	if err != nil {
+		t.Fatalf("newGatewayControlPlane: %v", err)
+	}
+	return cp
+}
+
+// Register carries the discovered GPU UUIDs as a STRINGSET resource, matching
+// the control plane's RegisterContainerInstance GPU parsing (Epic C3).
+func TestGatewayControlPlane_Register_SendsGPUStringSet(t *testing.T) {
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	id := identity{
+		ClusterName: "default", InstanceID: "i-1",
+		Capacity: detectCapacity([]string{"GPU-aaa", "GPU-bbb"}),
+	}
+	if err := cp.Register(id); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+
+	var in ecs.RegisterContainerInstanceInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	var gpu *ecs.Resource
+	for _, r := range in.TotalResources {
+		if r.Name != nil && *r.Name == "GPU" {
+			gpu = r
+		}
+	}
+	if gpu == nil {
+		t.Fatal("no GPU resource in TotalResources")
+	}
+	if *gpu.Type != "STRINGSET" {
+		t.Errorf("GPU resource type = %q, want STRINGSET", *gpu.Type)
+	}
+	got := make([]string, len(gpu.StringSetValue))
+	for i, v := range gpu.StringSetValue {
+		got[i] = *v
+	}
+	if len(got) != 2 || got[0] != "GPU-aaa" || got[1] != "GPU-bbb" {
+		t.Errorf("GPU UUIDs = %v, want [GPU-aaa GPU-bbb]", got)
+	}
+}
+
+// A non-GPU host (no discovered UUIDs) registers with no GPU resource at all.
+func TestGatewayControlPlane_Register_NoGPUOmitsResource(t *testing.T) {
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	id := identity{ClusterName: "default", InstanceID: "i-1", Capacity: detectCapacity(nil)}
+	if err := cp.Register(id); err != nil {
+		t.Fatalf("Register: %v", err)
+	}
+
+	var in ecs.RegisterContainerInstanceInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	for _, r := range in.TotalResources {
+		if r.Name != nil && *r.Name == "GPU" {
+			t.Fatalf("unexpected GPU resource on a non-GPU host: %+v", r)
+		}
+	}
+}
+
+// ReportTaskGPU posts the per-container device UUIDs under the ReportTaskGPU
+// action, decodable as the handlers_ecs internal shape.
+func TestGatewayControlPlane_ReportTaskGPU(t *testing.T) {
+	var gotTarget string
+	var body []byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotTarget = r.Header.Get("X-Amz-Target")
+		body, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	cp := newTestGatewayControlPlane(t, srv)
+	err := cp.ReportTaskGPU("web", "t-1", []handlers_ecs.ContainerGPUReport{
+		{Name: "trainer", GPUIDs: []string{"GPU-aaa"}},
+	})
+	if err != nil {
+		t.Fatalf("ReportTaskGPU: %v", err)
+	}
+	if gotTarget == "" {
+		t.Fatal("no X-Amz-Target header sent")
+	}
+
+	var in handlers_ecs.ReportTaskGPUInput
+	if err := json.Unmarshal(body, &in); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	if in.Cluster != "web" || in.Task != "t-1" {
+		t.Errorf("cluster/task = %q/%q, want web/t-1", in.Cluster, in.Task)
+	}
+	if len(in.Containers) != 1 || in.Containers[0].Name != "trainer" || len(in.Containers[0].GPUIDs) != 1 {
+		t.Errorf("containers = %+v, want one trainer entry with one UUID", in.Containers)
+	}
+}
@@ -192,6 +192,45 @@ func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Creden
 	return creds, roleARN, nil
 }

+// AssumeProvider returns a CredentialsProvider that mints credentials by assuming
+// roleARN over the gateway — the execution-role path for ECR image pulls. It
+// caches the assumed set and re-assumes within the refresh margin. session scopes
+// the STS session name.
+func (c *credEndpoint) AssumeProvider(roleARN, session string) credentials.CredentialsProvider {
+	return &assumeProvider{ep: c, roleARN: roleARN, session: session}
+}
+
+// assumeProvider adapts a credEndpoint's AssumeRole path to the agent's
+// CredentialsProvider interface, caching the assumed credentials.
+type assumeProvider struct {
+	ep      *credEndpoint
+	roleARN string
+	session string
+
+	mu     sync.Mutex
+	cached stsauth.Credentials
+}
+
+var _ credentials.CredentialsProvider = (*assumeProvider)(nil)
+
+func (p *assumeProvider) Retrieve(ctx context.Context) (credentials.Credentials, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if !credValid(p.cached, credRefreshMargin) {
+		creds, err := p.ep.assume(ctx, p.roleARN, p.session)
+		if err != nil {
+			return credentials.Credentials{}, err
+		}
+		p.cached = creds
+	}
+	return credentials.Credentials{
+		AccessKeyID:     p.cached.AccessKeyID,
+		SecretAccessKey: p.cached.SecretAccessKey,
+		SessionToken:    p.cached.SessionToken,
+		Expiration:      p.cached.Expiration,
+	}, nil
+}
+
 // assumeOverGateway mints credentials for roleARN by SigV4-signing AssumeRole
 // against the gateway with the agent's instance credentials.
 func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
@@ -0,0 +1,42 @@
+package main
+
+import (
+	"os/exec"
+	"strings"
+)
+
+// commandRunner abstracts exec.Command so discovery is unit-testable without
+// shelling out. Production uses execCommandRunner; tests inject a fake.
+type commandRunner func(name string, args ...string) ([]byte, error)
+
+// execCommandRunner runs a real host command and returns its stdout.
+func execCommandRunner(name string, args ...string) ([]byte, error) {
+	return exec.Command(name, args...).Output()
+}
+
+// discoverNvidiaGPUs enumerates local NVIDIA GPU device UUIDs via nvidia-smi.
+// A missing binary or any run failure (no GPU host) yields an empty list, not
+// an error — the agent still registers with zero GPU capacity.
+func discoverNvidiaGPUs(run commandRunner) []string {
+	out, err := run("nvidia-smi", "--query-gpu=uuid", "--format=csv,noheader")
+	if err != nil {
+		return nil
+	}
+	return parseNvidiaSMIUUIDs(string(out))
+}
+
+// parseNvidiaSMIUUIDs splits nvidia-smi's newline-delimited UUID output,
+// dropping blank lines. Exported as a pure function so fixture strings can be
+// tested without exec.
+func parseNvidiaSMIUUIDs(out string) []string {
+	lines := strings.Split(strings.TrimSpace(out), "\n")
+	uuids := make([]string, 0, len(lines))
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		uuids = append(uuids, line)
+	}
+	return uuids
+}
@@ -0,0 +1,71 @@
+package main
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+)
+
+func TestParseNvidiaSMIUUIDs(t *testing.T) {
+	cases := []struct {
+		name string
+		in   string
+		want []string
+	}{
+		{"empty", "", nil},
+		{"whitespace only", "   \n\n  ", nil},
+		{"single", "GPU-11111111-1111-1111-1111-111111111111\n", []string{"GPU-11111111-1111-1111-1111-111111111111"}},
+		{"multi", "GPU-aaa\nGPU-bbb\nGPU-ccc\n", []string{"GPU-aaa", "GPU-bbb", "GPU-ccc"}},
+		{"blank lines between", "GPU-aaa\n\nGPU-bbb\n", []string{"GPU-aaa", "GPU-bbb"}},
+		{"trailing whitespace per line", "GPU-aaa \r\nGPU-bbb\r\n", []string{"GPU-aaa", "GPU-bbb"}},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got := parseNvidiaSMIUUIDs(c.in)
+			if !reflect.DeepEqual(got, c.want) && (len(got) != 0 || len(c.want) != 0) {
+				t.Errorf("parseNvidiaSMIUUIDs(%q) = %v, want %v", c.in, got, c.want)
+			}
+		})
+	}
+}
+
+func TestDiscoverNvidiaGPUs_AbsentBinaryReturnsEmptyNoError(t *testing.T) {
+	run := func(name string, args ...string) ([]byte, error) {
+		return nil, errors.New(`exec: "nvidia-smi": executable file not found in $PATH`)
+	}
+	got := discoverNvidiaGPUs(run)
+	if len(got) != 0 {
+		t.Errorf("want empty list on absent nvidia-smi, got %v", got)
+	}
+}
+
+func TestDiscoverNvidiaGPUs_ParsesFixtureOutput(t *testing.T) {
+	var gotName string
+	var gotArgs []string
+	run := func(name string, args ...string) ([]byte, error) {
+		gotName = name
+		gotArgs = args
+		return []byte("GPU-aaa\nGPU-bbb\n"), nil
+	}
+	got := discoverNvidiaGPUs(run)
+	want := []string{"GPU-aaa", "GPU-bbb"}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+	if gotName != "nvidia-smi" {
+		t.Errorf("command = %q, want nvidia-smi", gotName)
+	}
+	if len(gotArgs) != 2 || gotArgs[0] != "--query-gpu=uuid" || gotArgs[1] != "--format=csv,noheader" {
+		t.Errorf("args = %v, want the uuid/csv,noheader query", gotArgs)
+	}
+}
+
+func TestDiscoverNvidiaGPUs_NoDevicesReturnsEmpty(t *testing.T) {
+	run := func(name string, args ...string) ([]byte, error) {
+		return []byte(""), nil
+	}
+	got := discoverNvidiaGPUs(run)
+	if len(got) != 0 {
+		t.Errorf("want empty list, got %v", got)
+	}
+}
@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+)
+
+// gpuLedger is the agent's local source of truth for which of the host's
+// discovered GPU UUIDs are free vs pinned to a running task's container. C4
+// reads a task's pinned UUIDs to build the CDI device list for containerd.
+type gpuLedger struct {
+	mu     sync.Mutex
+	free   []string
+	pinned map[string][]string // key ("taskID/containerName") -> uuids
+}
+
+// newGPULedger seeds the ledger with the host's discovered device UUIDs. A nil
+// or empty slice is a valid non-GPU host: every Pin then fails, which callers
+// treat as best-effort (no device, no CDI injection later).
+func newGPULedger(uuids []string) *gpuLedger {
+	free := append([]string(nil), uuids...)
+	return &gpuLedger{free: free, pinned: make(map[string][]string)}
+}
+
+// gpuKey builds the ledger key for one container's pin/release.
+func gpuKey(taskID, containerName string) string {
+	return taskID + "/" + containerName
+}
+
+// Pin reserves n free UUIDs under key, returning them. An error means fewer
+// than n devices were free; the key is left unpinned so a retry (or a
+// best-effort caller) can proceed without a partial reservation.
+func (l *gpuLedger) Pin(key string, n int) ([]string, error) {
+	if n <= 0 {
+		return nil, nil
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if len(l.free) < n {
+		return nil, fmt.Errorf("gpu ledger: requested %d GPUs, %d free", n, len(l.free))
+	}
+	uuids := append([]string(nil), l.free[:n]...)
+	l.free = l.free[n:]
+	l.pinned[key] = uuids
+	return uuids, nil
+}
+
+// ReleaseTask frees every UUID pinned under any container key belonging to
+// taskID, called once the task's terminal STOPPED state is reported.
+func (l *gpuLedger) ReleaseTask(taskID string) {
+	prefix := taskID + "/"
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	for key, uuids := range l.pinned {
+		if !strings.HasPrefix(key, prefix) {
+			continue
+		}
+		l.free = append(l.free, uuids...)
+		delete(l.pinned, key)
+	}
+}
@@ -0,0 +1,155 @@
+package main
+
+import (
+	"sync"
+	"testing"
+)
+
+func TestGPULedger_PinReturnsDistinctUUIDs(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b", "GPU-c"})
+	got, err := l.Pin("t-1/web", 2)
+	if err != nil {
+		t.Fatalf("Pin: %v", err)
+	}
+	if len(got) != 2 {
+		t.Fatalf("want 2 UUIDs, got %v", got)
+	}
+	if got[0] == got[1] {
+		t.Errorf("pinned UUIDs must be distinct, got %v", got)
+	}
+	if len(l.free) != 1 {
+		t.Errorf("free = %d, want 1 remaining", len(l.free))
+	}
+}
+
+func TestGPULedger_PinInsufficientFreeErrors(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a"})
+	_, err := l.Pin("t-1/web", 2)
+	if err == nil {
+		t.Fatal("want error when requesting more GPUs than free")
+	}
+	if len(l.free) != 1 {
+		t.Errorf("failed pin must not consume any device, free = %d", len(l.free))
+	}
+}
+
+func TestGPULedger_PinZeroIsNoop(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a"})
+	got, err := l.Pin("t-1/web", 0)
+	if err != nil || got != nil {
+		t.Errorf("Pin(0) = %v, %v; want nil, nil", got, err)
+	}
+}
+
+func TestGPULedger_ReleaseTaskFreesOnlyThatTasksDevices(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b", "GPU-c", "GPU-d"})
+	if _, err := l.Pin(gpuKey("t-1", "web"), 2); err != nil {
+		t.Fatalf("pin t-1: %v", err)
+	}
+	if _, err := l.Pin(gpuKey("t-2", "worker"), 1); err != nil {
+		t.Fatalf("pin t-2: %v", err)
+	}
+	if len(l.free) != 1 {
+		t.Fatalf("free = %d, want 1 before release", len(l.free))
+	}
+
+	l.ReleaseTask("t-1")
+	if len(l.free) != 3 {
+		t.Errorf("free = %d, want 3 after releasing t-1's two devices", len(l.free))
+	}
+	if _, ok := l.pinned[gpuKey("t-2", "worker")]; !ok {
+		t.Error("t-2's pin should survive releasing t-1")
+	}
+	if _, ok := l.pinned[gpuKey("t-1", "web")]; ok {
+		t.Error("t-1's pin should be gone after release")
+	}
+}
+
+// Multi-device pinning across concurrent task assigns must hand out disjoint
+// UUID sets — no two tasks may ever share a device.
+func TestGPULedger_ConcurrentPinsHandOutDistinctUUIDs(t *testing.T) {
+	const nDevices = 20
+	uuids := make([]string, nDevices)
+	for i := range uuids {
+		uuids[i] = "GPU-" + string(rune('a'+i))
+	}
+	l := newGPULedger(uuids)
+
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	seen := make(map[string]bool)
+	errs := 0
+	for i := range nDevices {
+		wg.Add(1)
+		go func(n int) {
+			defer wg.Done()
+			key := gpuKey("t-concurrent", string(rune('A'+n)))
+			got, err := l.Pin(key, 1)
+			mu.Lock()
+			defer mu.Unlock()
+			if err != nil {
+				errs++
+				return
+			}
+			for _, u := range got {
+				if seen[u] {
+					t.Errorf("UUID %s handed out to more than one concurrent pin", u)
+				}
+				seen[u] = true
+			}
+		}(i)
+	}
+	wg.Wait()
+	if errs != 0 {
+		t.Errorf("want all %d pins to succeed (exactly enough devices), got %d errors", nDevices, errs)
+	}
+	if len(seen) != nDevices {
+		t.Errorf("distinct UUIDs handed out = %d, want %d", len(seen), nDevices)
+	}
+	if len(l.free) != 0 {
+		t.Errorf("free = %d, want 0 after all devices pinned", len(l.free))
+	}
+}
+
+// Releasing a task's devices concurrently with new pins never double-hands a
+// UUID: a device is only ever free or pinned to exactly one key at a time.
+func TestGPULedger_ConcurrentPinAndReleaseNoDoubleAssignment(t *testing.T) {
+	l := newGPULedger([]string{"GPU-a", "GPU-b"})
+	var wg sync.WaitGroup
+	for round := range 50 {
+		wg.Add(2)
+		taskID := "t-" + string(rune('a'+round%26))
+		go func(id string) {
+			defer wg.Done()
+			_, _ = l.Pin(gpuKey(id, "web"), 1)
+		}(taskID)
+		go func(id string) {
+			defer wg.Done()
+			l.ReleaseTask(id)
+		}(taskID)
+	}
+	wg.Wait()
+	// Invariant: every currently-pinned UUID appears in exactly one pinned entry
+	// and never simultaneously in free.
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	freeSet := make(map[string]bool, len(l.free))
+	for _, u := range l.free {
+		if freeSet[u] {
+			t.Errorf("duplicate UUID %s in free list", u)
+		}
+		freeSet[u] = true
+	}
+	pinnedSet := make(map[string]bool)
+	for _, uuids := range l.pinned {
+		for _, u := range uuids {
+			if pinnedSet[u] {
+				t.Errorf("UUID %s pinned under more than one key", u)
+			}
+			if freeSet[u] {
+				t.Errorf("UUID %s is both free and pinned", u)
+			}
+			pinnedSet[u] = true
+		}
+	}
+}
@@ -7,24 +7,36 @@ import (
 	"testing"
 	"time"

+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )

-// fakeCP is a test double for the gateway control-plane. It records registers and
-// task-state reports, and replays a scripted queue of poll responses.
+// fakeCP is a test double for the gateway control-plane. It records registers,
+// task-state reports and GPU reports, and replays a scripted queue of poll
+// responses.
 type fakeCP struct {
 	mu sync.Mutex

-	registers   int
-	states      []bus.TaskState
-	pollAcks    [][]string
-	pollReplies [][]bus.Assign
-	pollCalls   int
+	registers    int
+	states       []bus.TaskState
+	gpuReports   []gpuReport
+	pollAcks     [][]string
+	pollStopAcks [][]string
+	pollReplies  [][]bus.Assign
+	stopReplies  [][]bus.StopDirective
+	pollCalls    int

 	registerErr bool
 	submitErr   bool
 }

+// gpuReport is one recorded ReportTaskGPU call.
+type gpuReport struct {
+	cluster    string
+	task       string
+	containers []handlers_ecs.ContainerGPUReport
+}
+
 var _ controlPlane = (*fakeCP)(nil)

 func (f *fakeCP) Register(_ identity) error {
@@ -47,17 +59,40 @@ func (f *fakeCP) SubmitTaskState(st bus.TaskState) error {
 	return nil
 }

-func (f *fakeCP) PollAssignments(_, _ string, ack []string) ([]bus.Assign, error) {
+func (f *fakeCP) PollAssignments(_, _ string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	cp := append([]string(nil), ack...)
-	f.pollAcks = append(f.pollAcks, cp)
+	f.pollAcks = append(f.pollAcks, append([]string(nil), ackAssigns...))
+	f.pollStopAcks = append(f.pollStopAcks, append([]string(nil), ackStops...))
 	var out []bus.Assign
 	if f.pollCalls < len(f.pollReplies) {
 		out = f.pollReplies[f.pollCalls]
 	}
+	var stops []bus.StopDirective
+	if f.pollCalls < len(f.stopReplies) {
+		stops = f.stopReplies[f.pollCalls]
+	}
 	f.pollCalls++
-	return out, nil
+	return out, stops, nil
+}
+
+func (f *fakeCP) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.gpuReports = append(f.gpuReports, gpuReport{cluster: cluster, task: task, containers: containers})
+	return nil
+}
+
+func (f *fakeCP) gpuReportsFor(taskID string) []gpuReport {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []gpuReport
+	for _, r := range f.gpuReports {
+		if r.task == taskID {
+			out = append(out, r)
+		}
+	}
+	return out
 }

 func (f *fakeCP) registerCount() int {
@@ -78,6 +113,12 @@ func (f *fakeCP) acks() [][]string {
 	return append([][]string(nil), f.pollAcks...)
 }

+func (f *fakeCP) stopAcks() [][]string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([][]string(nil), f.pollStopAcks...)
+}
+
 func testIdentity() identity {
 	return identity{
 		AccountID:    "123456789012",
@@ -90,6 +131,15 @@ func testIdentity() identity {
 	}
 }

+// testIdentityWithGPUs is testIdentity seeded with discovered GPU UUIDs, so
+// newAgent's ledger has real devices to pin from.
+func testIdentityWithGPUs(uuids ...string) identity {
+	id := testIdentity()
+	id.Capacity.GPU = len(uuids)
+	id.Capacity.GPUIDs = uuids
+	return id
+}
+
 func TestRegistrar_Register(t *testing.T) {
 	cp := &fakeCP{}
 	if err := newRegistrar(cp, testIdentity()).Register(); err != nil {
@@ -6,12 +6,41 @@ import (
 	"sort"

 	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/pkg/cdi"
 	"github.com/containerd/containerd/v2/pkg/cio"
 	"github.com/containerd/containerd/v2/pkg/namespaces"
 	"github.com/containerd/containerd/v2/pkg/oci"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )

+// cdiVendorGPU is the NVIDIA CDI vendor/class prefix for whole-GPU devices, per
+// the NVIDIA CDI naming convention (nvidia.com/gpu=<uuid>).
+const cdiVendorGPU = "nvidia.com/gpu"
+
+// cdiDeviceNames maps a container's pinned GPU device UUIDs to CDI device
+// names. Empty input (non-GPU container) returns nil so no CDI opt is added.
+func cdiDeviceNames(uuids []string) []string {
+	if len(uuids) == 0 {
+		return nil
+	}
+	names := make([]string, len(uuids))
+	for i, id := range uuids {
+		names[i] = cdiVendorGPU + "=" + id
+	}
+	return names
+}
+
+// cdiSpecOpts returns the oci.SpecOpts needed to inject gpuIDs (via
+// containerd's CDI machinery) as devices into a container's OCI spec. It
+// returns nil for a non-GPU container so Run adds nothing to the opts list.
+func cdiSpecOpts(gpuIDs []string) []oci.SpecOpts {
+	devices := cdiDeviceNames(gpuIDs)
+	if len(devices) == 0 {
+		return nil
+	}
+	return []oci.SpecOpts{cdi.WithCDIDevices(devices...)}
+}
+
 var _ Runner = (*containerdPuller)(nil)

 // Run creates a container from an already-pulled image and starts its task. The
@@ -51,6 +80,7 @@ func (p *containerdPuller) Run(ctx context.Context, id string, spec RunSpec) (st
 	if len(spec.Env) > 0 {
 		specOpts = append(specOpts, oci.WithEnv(envSlice(spec.Env)))
 	}
+	specOpts = append(specOpts, cdiSpecOpts(spec.GPUIDs)...)

 	container, err := p.client.NewContainer(ctx, id,
 		containerd.WithNewSnapshot(id+"-snapshot", image),
@@ -0,0 +1,47 @@
+package runtime
+
+import (
+	"reflect"
+	"testing"
+)
+
+// cdiDeviceNames maps a container's pinned GPU UUIDs to CDI device names
+// (nvidia.com/gpu=<uuid>); Run wires the result into oci.WithCDIDevices when
+// non-empty, so this is the injection logic's unit-testable core.
+func TestCDIDeviceNames(t *testing.T) {
+	cases := []struct {
+		name  string
+		uuids []string
+		want  []string
+	}{
+		{name: "empty", uuids: nil, want: nil},
+		{name: "single", uuids: []string{"GPU-aaaaaaaa-1111-2222-3333-444444444444"},
+			want: []string{"nvidia.com/gpu=GPU-aaaaaaaa-1111-2222-3333-444444444444"}},
+		{name: "multiple preserves order", uuids: []string{"GPU-aaa", "GPU-bbb"},
+			want: []string{"nvidia.com/gpu=GPU-aaa", "nvidia.com/gpu=GPU-bbb"}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := cdiDeviceNames(tc.uuids)
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("cdiDeviceNames(%v) = %v, want %v", tc.uuids, got, tc.want)
+			}
+		})
+	}
+}
+
+// TestCDISpecOpts_Wiring confirms Run's specOpts list picks up exactly one CDI
+// opt when GPUIDs is non-empty, and none for a non-GPU container — the wiring
+// Run relies on (a live oci.Spec is needed to unit test the opt's effect).
+func TestCDISpecOpts_Wiring(t *testing.T) {
+	if opts := cdiSpecOpts(nil); len(opts) != 0 {
+		t.Errorf("non-GPU container: want no spec opts, got %d", len(opts))
+	}
+	opts := cdiSpecOpts([]string{"GPU-aaa", "GPU-bbb"})
+	if len(opts) != 1 {
+		t.Fatalf("GPU container: want exactly one CDI spec opt, got %d", len(opts))
+	}
+	if opts[0] == nil {
+		t.Error("GPU container: spec opt is nil")
+	}
+}
@@ -48,6 +48,12 @@ type RunSpec struct {
 	// (awsvpc task ENI). Empty keeps the container in the host (VM) netns —
 	// bridge/host mode behaviour.
 	NetnsPath string
+	// GPU is the whole-GPU count requested for this container.
+	GPU int
+	// GPUIDs are the ledger-pinned device UUIDs assigned to this container, one
+	// per GPU requested. Empty for non-GPU containers or if pinning fell short.
+	// The runner injects these as CDI devices into the OCI spec.
+	GPUIDs []string
 }

 // RunStatus is a finished container's outcome.
@@ -8,6 +8,7 @@ import (
 	"time"

 	ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
+	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
 	"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
 )

@@ -22,31 +23,78 @@ func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool)
 	ticker := time.NewTicker(a.cfg.PollInterval)
 	defer ticker.Stop()

-	var ackNext []string
+	stopping := map[string]bool{}
+	var ackAssigns, ackStops []string
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-ticker.C:
-			assigns, err := a.cp.PollAssignments(a.id.ClusterName, a.id.InstanceID, ackNext)
+			assigns, stops, err := a.cp.PollAssignments(a.id.ClusterName, a.id.InstanceID, ackAssigns, ackStops)
 			if err != nil {
 				slog.Warn("ecs-agent: poll assignments failed", "err", err)
 				continue
 			}
-			ackNext = ackNext[:0]
+			// Dispatch stops first so a task stopped in the same poll it was assigned
+			// is never started.
+			ackStops = ackStops[:0]
+			for i := range stops {
+				sd := stops[i]
+				if !stopping[sd.TaskID] {
+					stopping[sd.TaskID] = true
+					slog.Info("ecs-agent: task stop requested", "task", sd.TaskID, "reason", sd.Reason)
+					go a.stopTask(ctx, sd)
+				}
+				ackStops = append(ackStops, sd.TaskID)
+			}
+			ackAssigns = ackAssigns[:0]
 			for i := range assigns {
 				as := assigns[i]
-				if !dispatched[as.TaskID] {
+				if !dispatched[as.TaskID] && !stopping[as.TaskID] {
 					dispatched[as.TaskID] = true
 					slog.Info("ecs-agent: task assigned", "task", as.TaskID, "containers", len(as.Containers))
 					go a.runTask(ctx, &as)
 				}
-				ackNext = append(ackNext, as.TaskID)
+				ackAssigns = append(ackAssigns, as.TaskID)
 			}
 		}
 	}
 }

+// stopTask reaps a task's containers on a control-plane stop directive: it lists
+// the runtime's containers, removes the ones labeled with this task (kill +
+// delete, releasing host ports/netns), then reports the task STOPPED with the
+// directive's reason. Idempotent — a task with no live containers still reports
+// STOPPED so the scheduler releases its capacity.
+func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) {
+	if a.runner == nil {
+		return
+	}
+	containers, err := a.runner.List(ctx)
+	if err != nil {
+		slog.Warn("ecs-agent: stop list failed", "task", sd.TaskID, "err", err)
+		return
+	}
+	statuses := make([]bus.ContainerStatus, 0)
+	for _, c := range containers {
+		if c.Labels[labelTaskID] != sd.TaskID {
+			continue
+		}
+		if rerr := a.runner.Remove(ctx, c.ID); rerr != nil {
+			slog.Warn("ecs-agent: stop remove failed", "task", sd.TaskID, "container", c.ID, "err", rerr)
+		}
+		statuses = append(statuses, bus.ContainerStatus{
+			Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
+		})
+	}
+	reason := sd.Reason
+	if reason == "" {
+		reason = "Task stopped"
+	}
+	a.reportTaskState(&bus.Assign{TaskID: sd.TaskID}, bus.TaskStatusStopped, reason, statuses)
+	slog.Info("ecs-agent: task reaped", "task", sd.TaskID, "containers", len(statuses))
+}
+
 // runTask pulls each container image, starts the containers, and reports task
 // state on the bus. A missing runtime or any per-container failure reports the
 // task STOPPED with a reason; success reports RUNNING and waits for exit.
@@ -65,10 +113,11 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 	}

 	credID := a.registerTaskCreds(as)
+	resolver := a.pullResolver(as)

 	statuses := make([]bus.ContainerStatus, 0, len(as.Containers))
 	for _, c := range as.Containers {
-		if _, err := a.puller.Pull(ctx, ctrruntime.PullSpec{Ref: c.Image}, a.resolver); err != nil {
+		if _, err := a.puller.Pull(ctx, ctrruntime.PullSpec{Ref: c.Image}, resolver); err != nil {
 			slog.Error("ecs-agent: pull failed", "task", as.TaskID, "image", c.Image, "err", err)
 			a.teardownTaskNetns(as)
 			a.reportTaskState(as, bus.TaskStatusStopped, "image pull failed: "+err.Error(), statuses)
@@ -76,12 +125,15 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 		}

 		cid := containerID(as.TaskID, c.Name)
+		gpuIDs := a.pinContainerGPUs(as.TaskID, c.Name, c.GPU)
 		spec := ctrruntime.RunSpec{
 			Image:     c.Image,
 			Command:   c.Command,
 			Env:       withCredEnv(c.Environment, credID),
 			Labels:    taskLabels(as, c.Name),
 			NetnsPath: netnsPath,
+			GPU:       c.GPU,
+			GPUIDs:    gpuIDs,
 		}
 		id, err := a.runner.Run(ctx, cid, spec)
 		if err != nil {
@@ -90,7 +142,9 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 			a.reportTaskState(as, bus.TaskStatusStopped, "container start failed: "+err.Error(), statuses)
 			return
 		}
-		statuses = append(statuses, bus.ContainerStatus{Name: c.Name, Status: bus.TaskStatusRunning, ContainerID: id})
+		statuses = append(statuses, bus.ContainerStatus{
+			Name: c.Name, Status: bus.TaskStatusRunning, ContainerID: id, GPUIDs: gpuIDs,
+		})
 		go a.waitContainer(ctx, as, c.Name, id)
 	}

@@ -137,6 +191,18 @@ func (a *Agent) teardownTaskNetns(as *bus.Assign) {
 	}
 }

+// pullResolver returns the ECR resolver used for a task's image pulls. When the
+// assign carries an execution role, it authorizes pulls as that role (assumed
+// over the gateway); an empty role or a nil credential endpoint (unit tests)
+// falls back to the instance-role resolver, so a pull is never worse off.
+func (a *Agent) pullResolver(as *bus.Assign) ctrruntime.Resolver {
+	if as.ExecutionRoleARN == "" || a.cred == nil {
+		return a.resolver
+	}
+	prov := a.cred.AssumeProvider(as.ExecutionRoleARN, sessionName(as.TaskID))
+	return newLazyECRResolver(prov, a.cfg.Region, a.cfg.GatewayURL, a.cfg.GatewayCA)
+}
+
 // registerTaskCreds registers the task's credID -> role mapping with the
 // credential endpoint and returns the credID (empty when the task has no role).
 func (a *Agent) registerTaskCreds(as *bus.Assign) string {
@@ -174,8 +240,26 @@ func withCredEnv(env map[string]string, credID string) map[string]string {
 	return out
 }

+// pinContainerGPUs reserves n device UUIDs for a container from the local
+// ledger, before the container is started so the runner can inject them as
+// CDI devices. A short ledger (fewer free devices than requested, e.g.
+// discovery found none) is logged and treated as best-effort: the container
+// still runs without pinned UUIDs (and without GPU access) rather than
+// failing the task.
+func (a *Agent) pinContainerGPUs(taskID, containerName string, n int) []string {
+	if n <= 0 || a.gpu == nil {
+		return nil
+	}
+	uuids, err := a.gpu.Pin(gpuKey(taskID, containerName), n)
+	if err != nil {
+		slog.Warn("ecs-agent: gpu pin short", "task", taskID, "container", containerName, "err", err)
+	}
+	return uuids
+}
+
 // reportTaskState reports a task transition through the gateway's
-// SubmitTaskStateChange action.
+// SubmitTaskStateChange action, then (RUNNING with pinned GPUs) reports the
+// per-container device UUIDs, and (STOPPED) releases them back to the ledger.
 func (a *Agent) reportTaskState(as *bus.Assign, status, reason string, containers []bus.ContainerStatus) {
 	if a.cp == nil {
 		return
@@ -193,6 +277,28 @@ func (a *Agent) reportTaskState(as *bus.Assign, status, reason string, container
 	if err := a.cp.SubmitTaskState(msg); err != nil {
 		slog.Error("ecs-agent: report task-state failed", "task", as.TaskID, "err", err)
 	}
+	a.reportTaskGPUs(as.TaskID, containers)
+	if status == bus.TaskStatusStopped && a.gpu != nil {
+		a.gpu.ReleaseTask(as.TaskID)
+	}
+}
+
+// reportTaskGPUs sends the control plane the pinned device UUIDs for any
+// container in the report that has them, so DescribeTasks can surface gpuIds.
+// A no-op when nothing was pinned (the common non-GPU-task case).
+func (a *Agent) reportTaskGPUs(taskID string, containers []bus.ContainerStatus) {
+	reports := make([]handlers_ecs.ContainerGPUReport, 0, len(containers))
+	for _, c := range containers {
+		if len(c.GPUIDs) > 0 {
+			reports = append(reports, handlers_ecs.ContainerGPUReport{Name: c.Name, GPUIDs: c.GPUIDs})
+		}
+	}
+	if len(reports) == 0 {
+		return
+	}
+	if err := a.cp.ReportTaskGPU(a.id.ClusterName, taskID, reports); err != nil {
+		slog.Error("ecs-agent: report task-gpu failed", "task", taskID, "err", err)
+	}
 }

 // containerID composes a containerd-valid container ID from a task + container.
@@ -70,6 +70,160 @@ func TestRunTask_PullRunReportsRunning(t *testing.T) {
 	}
 }

+// TestRunTask_GPUCarriedToRunSpec covers mulga-11opz (Epic C task C1): the
+// AssignContainer's GPU count reaches the runtime RunSpec unchanged; a
+// container with no GPU request keeps RunSpec.GPU at zero (regression).
+func TestRunTask_GPUCarriedToRunSpec(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 2,
+	})
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 2 {
+		t.Fatalf("expected two runs, got %+v", rt.Runs)
+	}
+	if rt.Runs[0].GPU != 0 {
+		t.Errorf("web container: want GPU=0, got %d", rt.Runs[0].GPU)
+	}
+	if rt.Runs[1].GPU != 2 {
+		t.Errorf("trainer container: want GPU=2, got %d", rt.Runs[1].GPU)
+	}
+}
+
+// TestRunTask_GPUPinsAndReportsUUIDs covers C3: a container's GPU request is
+// pinned from the agent's local ledger, carried on the RUNNING report's
+// ContainerStatus, and forwarded to the control plane via ReportTaskGPU.
+func TestRunTask_GPUPinsAndReportsUUIDs(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
+	})
+	a.runTask(context.Background(), as)
+
+	st := cp.taskStates()
+	if len(st) == 0 || st[len(st)-1].LastStatus != bus.TaskStatusRunning {
+		t.Fatalf("want RUNNING final state, got %+v", st)
+	}
+	last := st[len(st)-1]
+	var trainer *bus.ContainerStatus
+	for i := range last.Containers {
+		if last.Containers[i].Name == "trainer" {
+			trainer = &last.Containers[i]
+		}
+	}
+	if trainer == nil || len(trainer.GPUIDs) != 1 {
+		t.Fatalf("trainer container missing pinned GPU UUID: %+v", last.Containers)
+	}
+	if trainer.GPUIDs[0] != "GPU-aaa" && trainer.GPUIDs[0] != "GPU-bbb" {
+		t.Errorf("unexpected pinned UUID %q", trainer.GPUIDs[0])
+	}
+
+	reports := cp.gpuReportsFor(as.TaskID)
+	if len(reports) != 1 {
+		t.Fatalf("want one ReportTaskGPU call, got %d", len(reports))
+	}
+	if reports[0].cluster != as.ClusterName {
+		t.Errorf("report cluster = %q, want %q", reports[0].cluster, as.ClusterName)
+	}
+	if len(reports[0].containers) != 1 || reports[0].containers[0].Name != "trainer" {
+		t.Errorf("report containers = %+v, want one trainer entry", reports[0].containers)
+	}
+}
+
+// TestRunTask_GPUIDsReachRunSpec covers C4: the ledger's pinned UUIDs for a
+// container reach RunSpec.GPUIDs (not just the RunSpec.GPU count), so the
+// runner can inject them as CDI devices; a non-GPU container's RunSpec
+// carries no GPUIDs.
+func TestRunTask_GPUIDsReachRunSpec(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, rt, nil)
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 1,
+	})
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 2 {
+		t.Fatalf("expected two runs, got %+v", rt.Runs)
+	}
+	if len(rt.Runs[0].GPUIDs) != 0 {
+		t.Errorf("web container: want no GPUIDs, got %+v", rt.Runs[0].GPUIDs)
+	}
+	trainer := rt.Runs[1]
+	if len(trainer.GPUIDs) != 1 {
+		t.Fatalf("trainer container: want one pinned GPUID, got %+v", trainer.GPUIDs)
+	}
+	if trainer.GPUIDs[0] != "GPU-aaa" && trainer.GPUIDs[0] != "GPU-bbb" {
+		t.Errorf("unexpected pinned UUID %q", trainer.GPUIDs[0])
+	}
+}
+
+// A container with no GPU request never triggers a ReportTaskGPU call.
+func TestRunTask_NoGPUSkipsReport(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt)
+	as := testAssign()
+	a.runTask(context.Background(), as)
+
+	if reports := cp.gpuReportsFor(as.TaskID); len(reports) != 0 {
+		t.Errorf("want no GPU report for a non-GPU task, got %+v", reports)
+	}
+}
+
+// A GPU request exceeding the ledger's free devices (discovery found none, or
+// fewer than requested) is best-effort: the container still runs, it just
+// carries no pinned UUIDs and so no CDI devices are injected.
+func TestRunTask_GPUShortLedgerStillRuns(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
+	a := newRunAgent(cp, rt) // testIdentity() has no discovered GPU UUIDs
+	as := testAssign()
+	as.Containers = append(as.Containers, bus.AssignContainer{
+		Name: "trainer", Image: "registry/trainer:1", GPU: 2,
+	})
+	a.runTask(context.Background(), as)
+
+	if len(rt.Runs) != 2 {
+		t.Fatalf("expected both containers to run despite the short ledger, got %+v", rt.Runs)
+	}
+	if reports := cp.gpuReportsFor(as.TaskID); len(reports) != 0 {
+		t.Errorf("want no GPU report when nothing was pinned, got %+v", reports)
+	}
+}
+
+// stopTask releases the task's pinned GPU UUIDs back to the ledger so a later
+// task on the same instance can reuse them.
+func TestStopTask_ReleasesGPUsBackToLedger(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-trainer", Running: true, Labels: map[string]string{
+			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "trainer",
+		}},
+	}}
+	a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa"), cp, rt, rt, nil)
+	if _, err := a.gpu.Pin(gpuKey("t-001", "trainer"), 1); err != nil {
+		t.Fatalf("seed pin: %v", err)
+	}
+	if len(a.gpu.free) != 0 {
+		t.Fatalf("setup: want ledger fully pinned, free = %v", a.gpu.free)
+	}
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
+
+	if len(a.gpu.free) != 1 {
+		t.Errorf("want the GPU released back to the ledger, free = %v", a.gpu.free)
+	}
+}
+
 // A pull failure reports STOPPED and never starts the container.
 func TestRunTask_PullFailureReportsStopped(t *testing.T) {
 	cp := &fakeCP{}
@@ -172,6 +326,151 @@ func TestPollAssignments_DispatchesOnceAndAcks(t *testing.T) {
 	}
 }

+// stopTask reaps the task's labeled containers and reports STOPPED with the
+// directive reason and container status.
+func TestStopTask_ReapsLabeledContainersAndReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-web", Running: true, Labels: map[string]string{
+			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "web",
+		}},
+		{ID: "t-999-other", Running: true, Labels: map[string]string{
+			"mulga.ecs.taskID": "t-999", "mulga.ecs.containerName": "web",
+		}},
+	}}
+	a := newRunAgent(cp, rt)
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
+
+	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
+		t.Fatalf("want only t-001-web removed, got %+v", rt.Removed)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED report, got %+v", st)
+	}
+	if st[0].Reason != "bye" {
+		t.Errorf("want reason %q, got %q", "bye", st[0].Reason)
+	}
+	if len(st[0].Containers) != 1 || st[0].Containers[0].Status != bus.TaskStatusStopped {
+		t.Errorf("want one STOPPED container, got %+v", st[0].Containers)
+	}
+}
+
+// A stop for a task with no live containers still reports STOPPED so the
+// scheduler releases its capacity.
+func TestStopTask_NoContainersStillReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{}
+	a := newRunAgent(cp, rt)
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "gone"})
+
+	if len(rt.Removed) != 0 {
+		t.Errorf("nothing to remove, got %+v", rt.Removed)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED report, got %+v", st)
+	}
+}
+
+// A stop delivered by poll is dispatched once, acked in the stop-ack list, and
+// suppresses a same-poll assign for the same task.
+func TestPollAssignments_StopReapsAndSuppressesAssign(t *testing.T) {
+	as := testAssign()
+	cp := &fakeCP{
+		pollReplies: [][]bus.Assign{{*as}, {*as}, nil},
+		stopReplies: [][]bus.StopDirective{{{TaskID: as.TaskID, Reason: "bye"}}},
+	}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
+	}}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go a.pollAssignments(ctx, map[string]bool{})
+
+	deadline := time.After(time.Second)
+	for {
+		if stopped := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusStopped); stopped >= 1 {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatalf("stop never reaped; states=%+v", cp.taskStates())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+
+	// The assign for the stopping task must never have started a container.
+	if len(rt.Runs) != 0 {
+		t.Errorf("assign for a stopping task should not run, got %+v", rt.Runs)
+	}
+	if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 0 {
+		t.Errorf("stopping task should never report RUNNING, got %d", running)
+	}
+	acked := false
+	for _, ack := range cp.stopAcks() {
+		for _, id := range ack {
+			if id == as.TaskID {
+				acked = true
+			}
+		}
+	}
+	if !acked {
+		t.Errorf("stop %s was never acked; stopAcks=%v", as.TaskID, cp.stopAcks())
+	}
+}
+
+// stopTask with no runtime is a no-op: nothing to reap, no state reported.
+func TestStopTask_NilRunnerNoOp(t *testing.T) {
+	cp := &fakeCP{}
+	a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "x"})
+	if len(cp.taskStates()) != 0 {
+		t.Fatalf("nil runner should report no state, got %+v", cp.taskStates())
+	}
+}
+
+// A List failure aborts the reap without reporting a (false) STOPPED state.
+func TestStopTask_ListErrorNoReport(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{ListErr: errors.New("list boom")}
+	a := newRunAgent(cp, rt)
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "x"})
+	if len(cp.taskStates()) != 0 {
+		t.Fatalf("list error should report no state, got %+v", cp.taskStates())
+	}
+}
+
+// A directive with no reason reaps the matching container and reports STOPPED with
+// the default reason.
+func TestStopTask_EmptyReasonReapsWithDefault(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{{
+		ID: "t-001-web",
+		Labels: map[string]string{
+			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "web",
+		},
+	}}}
+	a := newRunAgent(cp, rt)
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001"})
+
+	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
+		t.Fatalf("want container t-001-web reaped, got %+v", rt.Removed)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED report, got %+v", st)
+	}
+	if st[0].Reason != "Task stopped" {
+		t.Errorf("want default reason %q, got %q", "Task stopped", st[0].Reason)
+	}
+}
+
 func countStatus(states []bus.TaskState, taskID, status string) int {
 	n := 0
 	for _, st := range states {
@@ -227,3 +526,32 @@ func TestContainerID(t *testing.T) {
 		t.Errorf("containerID = %q, want t-1-web", got)
 	}
 }
+
+// sentinelResolver is an identity-comparable Resolver for pullResolver tests.
+type sentinelResolver struct{}
+
+func (sentinelResolver) Authorize(context.Context, string) (string, string, string, error) {
+	return "", "", "", nil
+}
+
+// TestPullResolver covers siv-459: an assign with no execution role (or no
+// credential endpoint) uses the instance resolver; with both, a distinct
+// exec-role resolver is built.
+func TestPullResolver(t *testing.T) {
+	fake := sentinelResolver{}
+	a := newAgent(config{Region: "us-east-1"}, testIdentity(), &fakeCP{}, nil, nil, fake)
+
+	if got := a.pullResolver(&bus.Assign{TaskID: "t1"}); got != fake {
+		t.Fatalf("no execution role: want instance resolver fallback")
+	}
+	execAssign := &bus.Assign{TaskID: "t1", ExecutionRoleARN: "arn:aws:iam::1:role/e"}
+	if got := a.pullResolver(execAssign); got != fake {
+		t.Fatalf("nil cred endpoint: want instance resolver fallback")
+	}
+
+	a.cred = &credEndpoint{}
+	got := a.pullResolver(execAssign)
+	if got == nil || got == fake {
+		t.Fatalf("execution role + cred endpoint: want distinct exec-role resolver, got %v", got)
+	}
+}
@@ -17,10 +17,14 @@
 // Usage:
 //
 //	eks-gateway-fetch -resource addons   # GET /clusters/{cluster}/internal-addons/{accountId}
+//	eks-gateway-fetch -resource recovery -instance-id i-… # GET /clusters/{cluster}/internal-recovery/{accountId}/{instanceId}
+//
+// The recovery resource returns a single epoch\taction\tsnapshot line the on-VM
+// k3s-recovery agent applies before k3s starts (etcd cluster-reset / wipe-rejoin).
 //
 // Flags default to environment variables seeded by cloud-init:
 // EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
-// EKS_ACCOUNT_ID, EKS_CLUSTER_NAME.
+// EKS_ACCOUNT_ID, EKS_CLUSTER_NAME, EKS_INSTANCE_ID.
 package main

 import (
@@ -68,6 +72,7 @@ func main() {
 		accountID   string
 		clusterName string
 		resource    string
+		instanceID  string
 	)

 	flag.StringVar(&gatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
@@ -77,7 +82,8 @@ func main() {
 	flag.StringVar(&region, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
 	flag.StringVar(&accountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
 	flag.StringVar(&clusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
-	flag.StringVar(&resource, "resource", "addons", "Resource to fetch: addons")
+	flag.StringVar(&instanceID, "instance-id", os.Getenv("EKS_INSTANCE_ID"), "This member's EC2 instance ID (recovery resource)")
+	flag.StringVar(&resource, "resource", "addons", "Resource to fetch: addons | recovery")
 	flag.Parse()

 	switch {
@@ -85,15 +91,29 @@ func main() {
 		fatal("--account-id is required (or set EKS_ACCOUNT_ID)")
 	case clusterName == "":
 		fatal("--cluster is required (or set EKS_CLUSTER_NAME)")
-	case resource != "addons":
-		fatal("--resource must be addons")
+	case resource != "addons" && resource != "recovery":
+		fatal("--resource must be addons or recovery")
+	case resource == "recovery" && instanceID == "":
+		fatal("--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)")
 	}

 	client, err := eksgw.New(gatewayURL, gatewayCA, accessKey, secretKey, region)
 	if err != nil {
 		fatal(fmt.Sprintf("build gateway client: %v", err))
 	}
-	path := "/clusters/" + clusterName + "/internal-addons/" + accountID
+
+	var (
+		path string
+		emit func(io.Writer, []byte) error
+	)
+	switch resource {
+	case "recovery":
+		path = "/clusters/" + clusterName + "/internal-recovery/" + accountID + "/" + instanceID
+		emit = emitRecoveryTSV
+	default:
+		path = "/clusters/" + clusterName + "/internal-addons/" + accountID
+		emit = emitAddonsTSV
+	}

 	var lastErr error
 	for attempt := 1; attempt <= maxAttempts; attempt++ {
@@ -106,7 +126,7 @@ func main() {
 			}
 			continue
 		}
-		if werr := emitAddonsTSV(os.Stdout, body); werr != nil {
+		if werr := emit(os.Stdout, body); werr != nil {
 			fatal(fmt.Sprintf("render response: %v", werr))
 		}
 		return
@@ -133,6 +153,43 @@ func emitAddonsTSV(out io.Writer, body []byte) error {
 	return w.Flush()
 }

+// recoveryDirective mirrors the gateway's RecoveryDirective wire shape; duplicated
+// here to keep this tiny VM binary free of the handler package's dependency graph.
+type recoveryDirective struct {
+	Epoch            int64  `json:"epoch"`
+	Action           string `json:"action"`
+	Snapshot         string `json:"snapshot"`
+	SnapshotRequired bool   `json:"snapshotRequired"`
+}
+
+type internalRecoveryResponse struct {
+	Directive recoveryDirective `json:"directive"`
+}
+
+// emitRecoveryTSV decodes the recovery directive and writes a single
+// epoch\taction\tsnapshot\tsnapshotRequired line the on-VM k3s-recovery agent
+// consumes without a JSON parser. An unset action defaults to "none" (steady
+// state); snapshotRequired is 1 when the snapshot MUST restore (fresh DR seed
+// aborts boot on fetch failure) else 0.
+func emitRecoveryTSV(out io.Writer, body []byte) error {
+	var resp internalRecoveryResponse
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return fmt.Errorf("unmarshal internal-recovery response: %w", err)
+	}
+	d := resp.Directive
+	if d.Action == "" {
+		d.Action = "none"
+	}
+	required := "0"
+	if d.SnapshotRequired {
+		required = "1"
+	}
+	if _, err := fmt.Fprintf(out, "%d\t%s\t%s\t%s\n", d.Epoch, d.Action, d.Snapshot, required); err != nil {
+		return err
+	}
+	return nil
+}
+
 func fatal(msg string) {
 	slog.Error("eks-gateway-fetch: " + msg)
 	os.Exit(1)
@@ -0,0 +1,245 @@
+// Command eks-konnectivity-cert mints, then reuses, an ECDSA-P256 serving
+// certificate for the on-node konnectivity-server, signed by the K3s cluster CA.
+//
+// The konnectivity-agent (DaemonSet on workers) verifies the server it dials
+// using the in-pod ServiceAccount CA (/var/run/secrets/.../ca.crt), which in
+// K3s is the same server-ca that signs the apiserver. Signing the konnectivity
+// serving cert with that CA is therefore what lets the agent validate the
+// reverse-tunnel endpoint with no separate CA distribution.
+//
+// The eks-node AMI carries no openssl, so this is the Go equivalent: load the
+// K3s server CA cert+key, issue a leaf (ServerAuth) cert carrying the cluster's
+// konnectivity SANs (the NLB private-endpoint IP, public endpoint IP, NLB DNS),
+// and cache it under -dir so the OpenRC service is byte-stable across restarts.
+//
+// Usage:
+//
+//	eks-konnectivity-cert -dir /var/lib/spinifex-eks/konnectivity \
+//	    -ca-cert /var/lib/rancher/k3s/server/tls/server-ca.crt \
+//	    -ca-key  /var/lib/rancher/k3s/server/tls/server-ca.key \
+//	    -cn konnectivity-server -sans 10.32.100.4,203.0.113.9,eks-toc.example
+//
+// Writes <dir>/tls.crt and <dir>/tls.key; prints the two paths (tab-separated).
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"log/slog"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func main() {
+	dir := flag.String("dir", "", "directory to cache tls.crt/tls.key in")
+	caCert := flag.String("ca-cert", "", "PEM path of the signing (K3s server) CA cert")
+	caKey := flag.String("ca-key", "", "PEM path of the signing CA private key")
+	cn := flag.String("cn", "konnectivity-server", "certificate common name")
+	sansCSV := flag.String("sans", "", "comma-separated SANs (IP or DNS)")
+	flag.Parse()
+
+	if *dir == "" || *caCert == "" || *caKey == "" || *sansCSV == "" {
+		fatal("eks-konnectivity-cert: -dir, -ca-cert, -ca-key and -sans are required")
+	}
+	dns, ips := splitSANs(*sansCSV)
+	if len(dns) == 0 && len(ips) == 0 {
+		fatal("eks-konnectivity-cert: -sans lists no usable names")
+	}
+
+	certPath, keyPath, err := ensureCert(*dir, *caCert, *caKey, *cn, dns, ips)
+	if err != nil {
+		fatal("eks-konnectivity-cert: " + err.Error())
+	}
+	fmt.Printf("%s\t%s\n", certPath, keyPath)
+}
+
+// ensureCert returns the cached cert/key paths under dir, minting and persisting
+// a fresh CA-signed pair when none is present, readable, and still SAN-current.
+func ensureCert(dir, caCertPath, caKeyPath, cn string, dns []string, ips []net.IP) (string, string, error) {
+	certPath := filepath.Join(dir, "tls.crt")
+	keyPath := filepath.Join(dir, "tls.key")
+
+	if certCoversSANs(certPath, keyPath, dns, ips) {
+		return certPath, keyPath, nil
+	}
+
+	caCert, caKey, err := loadCA(caCertPath, caKeyPath)
+	if err != nil {
+		return "", "", err
+	}
+	certPEM, keyPEM, err := issueLeaf(caCert, caKey, cn, dns, ips)
+	if err != nil {
+		return "", "", err
+	}
+	if err = os.MkdirAll(dir, 0o700); err != nil {
+		return "", "", fmt.Errorf("mkdir %s: %w", dir, err)
+	}
+	if err = os.WriteFile(certPath, certPEM, 0o644); err != nil {
+		return "", "", fmt.Errorf("write %s: %w", certPath, err)
+	}
+	if err = os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
+		return "", "", fmt.Errorf("write %s: %w", keyPath, err)
+	}
+	return certPath, keyPath, nil
+}
+
+// certCoversSANs reports whether the cached cert exists, parses, has a key, and
+// already carries every requested SAN — so a cert minted before the endpoint
+// set changed is re-issued rather than served stale.
+func certCoversSANs(certPath, keyPath string, dns []string, ips []net.IP) bool {
+	c, err := os.ReadFile(certPath)
+	if err != nil {
+		return false
+	}
+	if k, kerr := os.ReadFile(keyPath); kerr != nil || len(k) == 0 {
+		return false
+	}
+	der := decodeFirst(c)
+	if der == nil {
+		return false
+	}
+	crt, err := x509.ParseCertificate(der)
+	if err != nil {
+		return false
+	}
+	have := make(map[string]struct{}, len(crt.DNSNames)+len(crt.IPAddresses))
+	for _, n := range crt.DNSNames {
+		have[n] = struct{}{}
+	}
+	for _, ip := range crt.IPAddresses {
+		have[ip.String()] = struct{}{}
+	}
+	for _, n := range dns {
+		if _, ok := have[n]; !ok {
+			return false
+		}
+	}
+	for _, ip := range ips {
+		if _, ok := have[ip.String()]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+// loadCA parses the PEM CA cert + EC/PKCS8 private key used to sign the leaf.
+func loadCA(certPath, keyPath string) (*x509.Certificate, *ecdsa.PrivateKey, error) {
+	certPEM, err := os.ReadFile(certPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("read ca cert %s: %w", certPath, err)
+	}
+	der := decodeFirst(certPEM)
+	if der == nil {
+		return nil, nil, fmt.Errorf("ca cert %s: no PEM block", certPath)
+	}
+	caCert, err := x509.ParseCertificate(der)
+	if err != nil {
+		return nil, nil, fmt.Errorf("parse ca cert: %w", err)
+	}
+
+	keyPEM, err := os.ReadFile(keyPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("read ca key %s: %w", keyPath, err)
+	}
+	keyDER := decodeFirst(keyPEM)
+	if keyDER == nil {
+		return nil, nil, fmt.Errorf("ca key %s: no PEM block", keyPath)
+	}
+	caKey, err := parseECKey(keyDER)
+	if err != nil {
+		return nil, nil, err
+	}
+	return caCert, caKey, nil
+}
+
+// parseECKey accepts the EC SEC1 or PKCS8 encodings K3s may write for the CA key.
+func parseECKey(der []byte) (*ecdsa.PrivateKey, error) {
+	if k, err := x509.ParseECPrivateKey(der); err == nil {
+		return k, nil
+	}
+	k8, err := x509.ParsePKCS8PrivateKey(der)
+	if err != nil {
+		return nil, fmt.Errorf("parse ca key (sec1 and pkcs8 failed): %w", err)
+	}
+	ec, ok := k8.(*ecdsa.PrivateKey)
+	if !ok {
+		return nil, fmt.Errorf("ca key is %T, want ecdsa", k8)
+	}
+	return ec, nil
+}
+
+// issueLeaf mints an ECDSA-P256 ServerAuth leaf for cn + SANs, signed by caCert/caKey.
+func issueLeaf(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, cn string, dns []string, ips []net.IP) (certPEM, keyPEM []byte, err error) {
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, nil, fmt.Errorf("generate key: %w", err)
+	}
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return nil, nil, fmt.Errorf("serial: %w", err)
+	}
+	now := time.Now().UTC()
+	tmpl := x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		DNSNames:              dns,
+		IPAddresses:           ips,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, caCert, &priv.PublicKey, caKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("create cert: %w", err)
+	}
+	keyDER, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, nil, fmt.Errorf("marshal key: %w", err)
+	}
+	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
+	return certPEM, keyPEM, nil
+}
+
+// splitSANs partitions the CSV into DNS names and IPs (an entry that parses as an
+// IP goes to IPAddresses, else DNSNames), matching how SAN matching is typed.
+func splitSANs(s string) (dns []string, ips []net.IP) {
+	for p := range strings.SplitSeq(s, ",") {
+		p = strings.TrimSpace(p)
+		if p == "" {
+			continue
+		}
+		if ip := net.ParseIP(p); ip != nil {
+			ips = append(ips, ip)
+			continue
+		}
+		dns = append(dns, p)
+	}
+	return dns, ips
+}
+
+// decodeFirst returns the DER bytes of the first PEM block, or nil.
+func decodeFirst(pemBytes []byte) []byte {
+	blk, _ := pem.Decode(pemBytes)
+	if blk == nil {
+		return nil
+	}
+	return blk.Bytes
+}
+
+func fatal(msg string) {
+	slog.Error(msg)
+	os.Exit(1)
+}
@@ -0,0 +1,153 @@
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// writeTestCA mints a self-signed EC CA (cert + SEC1 key PEM) into dir, returning
+// the two paths — a stand-in for K3s' /var/lib/rancher/k3s/server/tls/server-ca.*.
+func writeTestCA(t *testing.T, dir string) (certPath, keyPath string) {
+	t.Helper()
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("gen ca key: %v", err)
+	}
+	tmpl := x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "k3s-server-ca"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
+	if err != nil {
+		t.Fatalf("create ca cert: %v", err)
+	}
+	keyDER, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		t.Fatalf("marshal ca key: %v", err)
+	}
+	certPath = filepath.Join(dir, "server-ca.crt")
+	keyPath = filepath.Join(dir, "server-ca.key")
+	if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
+		t.Fatalf("write ca cert: %v", err)
+	}
+	if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
+		t.Fatalf("write ca key: %v", err)
+	}
+	return certPath, keyPath
+}
+
+func leafFrom(t *testing.T, certPath string) *x509.Certificate {
+	t.Helper()
+	b, err := os.ReadFile(certPath)
+	if err != nil {
+		t.Fatalf("read leaf: %v", err)
+	}
+	crt, err := x509.ParseCertificate(decodeFirst(b))
+	if err != nil {
+		t.Fatalf("parse leaf: %v", err)
+	}
+	return crt
+}
+
+func TestEnsureCertSignsLeafWithSANsThatChainsToCA(t *testing.T) {
+	dir := t.TempDir()
+	caCert, caKey := writeTestCA(t, dir)
+
+	dns, ips := splitSANs("10.32.100.4,203.0.113.9,eks-toc.example")
+	certPath, keyPath, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips)
+	if err != nil {
+		t.Fatalf("ensureCert: %v", err)
+	}
+	if _, err := os.Stat(keyPath); err != nil {
+		t.Fatalf("key not written: %v", err)
+	}
+
+	leaf := leafFrom(t, certPath)
+	if leaf.IsCA {
+		t.Error("konnectivity serving cert must be a leaf, not a CA")
+	}
+	// The agent dials the NLB private-endpoint IP and verifies via the in-pod CA,
+	// so the IP must be a SAN or the tunnel TLS handshake fails.
+	if err := leaf.VerifyHostname("10.32.100.4"); err != nil {
+		t.Errorf("IP SAN 10.32.100.4 missing: %v", err)
+	}
+
+	// The leaf must chain to the K3s server CA — that CA is the in-pod ca.crt the
+	// agent trusts, so no separate CA distribution is needed.
+	caPEM, _ := os.ReadFile(caCert)
+	roots := x509.NewCertPool()
+	if !roots.AppendCertsFromPEM(caPEM) {
+		t.Fatal("append CA failed")
+	}
+	if _, err := leaf.Verify(x509.VerifyOptions{
+		Roots:     roots,
+		DNSName:   "eks-toc.example",
+		KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}); err != nil {
+		t.Errorf("leaf does not chain to the K3s server CA: %v", err)
+	}
+}
+
+func TestEnsureCertReusesThenReissuesOnSANChange(t *testing.T) {
+	dir := t.TempDir()
+	caCert, caKey := writeTestCA(t, dir)
+
+	dns, ips := splitSANs("10.32.100.4,eks-toc.example")
+	c1, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips)
+	if err != nil {
+		t.Fatalf("first ensureCert: %v", err)
+	}
+	first, _ := os.ReadFile(c1)
+
+	// Same SANs → cached pair reused byte-for-byte (a churning cert would bounce
+	// the konnectivity-server on every restart).
+	if _, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips); err != nil {
+		t.Fatalf("second ensureCert: %v", err)
+	}
+	if second, _ := os.ReadFile(c1); string(first) != string(second) {
+		t.Error("ensureCert re-minted instead of reusing the SAN-current cached cert")
+	}
+
+	// A new endpoint in the SAN set must force a re-issue.
+	dns2, ips2 := splitSANs("10.32.100.4,eks-toc.example,10.32.100.99")
+	if _, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns2, ips2); err != nil {
+		t.Fatalf("third ensureCert: %v", err)
+	}
+	if third, _ := os.ReadFile(c1); string(first) == string(third) {
+		t.Error("ensureCert served a stale cert that does not cover the new SAN")
+	}
+	if err := leafFrom(t, c1).VerifyHostname("10.32.100.99"); err != nil {
+		t.Errorf("re-issued cert missing new SAN: %v", err)
+	}
+}
+
+func TestSplitSANsPartitionsIPsAndDNS(t *testing.T) {
+	dns, ips := splitSANs(" 10.0.0.1 , host.example , , 2001:db8::1 ")
+	if len(dns) != 1 || dns[0] != "host.example" {
+		t.Errorf("dns = %v, want [host.example]", dns)
+	}
+	wantIPs := []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("2001:db8::1")}
+	if len(ips) != len(wantIPs) {
+		t.Fatalf("ips = %v, want %v", ips, wantIPs)
+	}
+	for i, ip := range ips {
+		if !ip.Equal(wantIPs[i]) {
+			t.Errorf("ips[%d] = %v, want %v", i, ip, wantIPs[i])
+		}
+	}
+}
@@ -18,14 +18,10 @@ import (
 	"github.com/mulgadc/spinifex/internal/tlsconfig"
 )

-// tlsCertificate is the webhook's loopback serving certificate.
-type tlsCertificate struct {
-	cert tls.Certificate
-}
-
-func (c tlsCertificate) serverTLSConfig() *tls.Config {
+// serverTLSConfig is the webhook's loopback serving TLS config.
+func serverTLSConfig(cert tls.Certificate) *tls.Config {
 	return &tls.Config{
-		Certificates:     []tls.Certificate{c.cert},
+		Certificates:     []tls.Certificate{cert},
 		MinVersion:       tls.VersionTLS13,
 		CurvePreferences: tlsconfig.Curves,
 	}
@@ -36,30 +32,30 @@ func (c tlsCertificate) serverTLSConfig() *tls.Config {
 // across restarts keeps the CA the apiserver loaded into its webhook kubeconfig
 // valid even if the webhook process is restarted. Returns the parsed cert and
 // the certificate PEM (for embedding as the kubeconfig CA bundle).
-func ensureServingCert(certPath, keyPath string) (tlsCertificate, []byte, error) {
+func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error) {
 	if certPEM, keyPEM, ok := readCertPair(certPath, keyPath); ok {
 		cert, err := tls.X509KeyPair(certPEM, keyPEM)
 		if err == nil {
-			return tlsCertificate{cert: cert}, certPEM, nil
+			return cert, certPEM, nil
 		}
 		// Fall through to regenerate on a corrupt/incompatible persisted pair.
 	}

 	certPEM, keyPEM, err := generateSelfSigned()
 	if err != nil {
-		return tlsCertificate{}, nil, err
+		return tls.Certificate{}, nil, err
 	}
 	if err := os.WriteFile(certPath, certPEM, 0o644); err != nil {
-		return tlsCertificate{}, nil, fmt.Errorf("write cert %s: %w", certPath, err)
+		return tls.Certificate{}, nil, fmt.Errorf("write cert %s: %w", certPath, err)
 	}
 	if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
-		return tlsCertificate{}, nil, fmt.Errorf("write key %s: %w", keyPath, err)
+		return tls.Certificate{}, nil, fmt.Errorf("write key %s: %w", keyPath, err)
 	}
 	cert, err := tls.X509KeyPair(certPEM, keyPEM)
 	if err != nil {
-		return tlsCertificate{}, nil, fmt.Errorf("parse minted keypair: %w", err)
+		return tls.Certificate{}, nil, fmt.Errorf("parse minted keypair: %w", err)
 	}
-	return tlsCertificate{cert: cert}, certPEM, nil
+	return cert, certPEM, nil
 }

 func readCertPair(certPath, keyPath string) (certPEM, keyPEM []byte, ok bool) {
@@ -10,6 +10,9 @@
 // On startup it writes the apiserver webhook kubeconfig (with its self-signed
 // serving CA) so k3s can be pointed at it via
 // --authentication-token-webhook-config-file.
+//
+// SigV4 creds come from the AWS SDK chain (IMDS instance-role) when static
+// EKS_ACCESS_KEY/EKS_SECRET_KEY are absent, matching the sibling helpers.
 package main

 import (
@@ -56,11 +59,13 @@ func loadConfig() (config, error) {
 		keyPath:     envOr("EKS_WEBHOOK_KEY", "/etc/spinifex-eks/token-webhook.key"),
 		kubeconfig:  envOr("EKS_WEBHOOK_KUBECONFIG", "/etc/spinifex-eks/token-webhook.kubeconfig"),
 	}
+	// Static SigV4 creds are optional: when absent, eksgw.New signs with the
+	// AWS SDK chain (IMDS instance-role creds), the same path the CP VM's
+	// sibling helpers use. The CP VM launches on an instance profile, so
+	// buildK3sUserData omits EKS_ACCESS_KEY/EKS_SECRET_KEY by design.
 	switch {
 	case c.gatewayURL == "":
 		return c, errors.New("EKS_GATEWAY_URL not set")
-	case c.accessKey == "" || c.secretKey == "":
-		return c, errors.New("EKS_ACCESS_KEY / EKS_SECRET_KEY not set")
 	case c.accountID == "":
 		return c, errors.New("EKS_ACCOUNT_ID not set")
 	case c.clusterName == "":
@@ -125,7 +130,12 @@ func run(cfg config) error {
 	})
 	mux.HandleFunc("/authenticate", authr.handle)

-	server := newServer(cfg.addr, mux, tlsCert)
+	server := &http.Server{
+		Addr:              cfg.addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+		TLSConfig:         serverTLSConfig(tlsCert),
+	}
 	slog.Info("eks-token-webhook listening", "addr", cfg.addr, "cluster", cfg.clusterName)
 	if err := server.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
 		return fmt.Errorf("serve: %w", err)
@@ -214,14 +224,3 @@ func (a *authenticator) handle(w http.ResponseWriter, r *http.Request) {
 		slog.Error("encode TokenReview response", "err", err)
 	}
 }
-
-// newServer builds the loopback TLS server. Split out so tests can construct it
-// without binding a port.
-func newServer(addr string, handler http.Handler, cert tlsCertificate) *http.Server {
-	return &http.Server{
-		Addr:              addr,
-		Handler:           handler,
-		ReadHeaderTimeout: 5 * time.Second,
-		TLSConfig:         cert.serverTLSConfig(),
-	}
-}
@@ -99,7 +99,7 @@ func TestEnsureServingCert_PersistsAndReuses(t *testing.T) {
 	c1, pem1, err := ensureServingCert(certPath, keyPath)
 	require.NoError(t, err)
 	require.NotEmpty(t, pem1)
-	require.NotNil(t, c1.serverTLSConfig())
+	require.NotNil(t, serverTLSConfig(c1))

 	// Second call reuses the persisted pair (same cert bytes).
 	_, pem2, err := ensureServingCert(certPath, keyPath)
@@ -121,3 +121,41 @@ func TestWriteAPIServerKubeconfig(t *testing.T) {
 	assert.Contains(t, body, base64.StdEncoding.EncodeToString(certPEM))
 	assert.Contains(t, body, "current-context: webhook")
 }
+
+// The CP VM launches on an instance profile, so cloud-init omits the static
+// EKS_ACCESS_KEY/EKS_SECRET_KEY and eksgw.New signs with IMDS creds. loadConfig
+// must accept absent static keys; gating on them aborts every IMDS-mode boot,
+// leaving the apiserver webhook kubeconfig unwritten.
+func TestLoadConfig_AcceptsAbsentStaticCreds(t *testing.T) {
+	setEnv := func(t *testing.T, kv map[string]string) {
+		t.Helper()
+		for _, k := range []string{
+			"EKS_GATEWAY_URL", "EKS_ACCESS_KEY", "EKS_SECRET_KEY",
+			"EKS_ACCOUNT_ID", "EKS_CLUSTER_NAME",
+		} {
+			t.Setenv(k, "")
+		}
+		for k, v := range kv {
+			t.Setenv(k, v)
+		}
+	}
+
+	// IMDS mode: required vars set, static creds absent → accepted.
+	setEnv(t, map[string]string{
+		"EKS_GATEWAY_URL":  "https://gw:9999",
+		"EKS_ACCOUNT_ID":   "000000000001",
+		"EKS_CLUSTER_NAME": "toc",
+	})
+	cfg, err := loadConfig()
+	require.NoError(t, err)
+	assert.Empty(t, cfg.accessKey)
+	assert.Empty(t, cfg.secretKey)
+
+	// Genuinely-required vars are still enforced.
+	setEnv(t, map[string]string{
+		"EKS_ACCOUNT_ID":   "000000000001",
+		"EKS_CLUSTER_NAME": "toc",
+	})
+	_, err = loadConfig()
+	require.Error(t, err)
+}
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package autoinstall reads headless install parameters from environment
 // variables exported by spinifex-init.sh from the kernel cmdline.
 // Parameters are set in the GRUB "Headless" menu entry in grub.cfg.
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package branding centralises Mulga/Spinifex visual identity constants used
 // by the installer TUI.
 //
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package firstboot writes the oneshot systemd service and configuration that
 // completes Spinifex provisioning on the first real boot after installation.
 package firstboot
@@ -91,6 +74,31 @@ func writeScript(root string, cfg Config) error {
 		setupOVN += fmt.Sprintf(" --encap-ip=%s", cfg.EncapIP)
 	}

+	// Pre-start OVS and OVN central so their databases are initialised before
+	// setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes
+	// longer than setup-ovn.sh's internal 15-second timeout allows. Starting them
+	// here and waiting until the NB DB is ready means setup-ovn.sh sees a live DB
+	// the moment it starts — no races, no timeout failures.
+	ovnPrestart := `systemctl start openvswitch-switch
+systemctl start ovn-central
+echo "Waiting for OVN NB DB to initialise..."
+for _i in $(seq 1 120); do
+    if ovn-nbctl --timeout=2 get-connection >/dev/null 2>&1; then
+        echo "OVN NB DB ready (${_i}s)"
+        break
+    fi
+    sleep 1
+done`
+
+	// When a provisioning controller owns formation (SkipFormation) it also owns
+	// the clustered OVN bring-up via setup-ovn.sh's RAFT flags. firstboot must
+	// not start a standalone ovn-central here: the single-node .db it would leave
+	// blocks ovn-ctl's create/join (which require a clean DB). Defer both.
+	if cfg.SkipFormation {
+		ovnPrestart = `echo "[firstboot] OVN bring-up deferred to provisioning controller"`
+		setupOVN = `echo "[firstboot] setup-ovn deferred to provisioning controller"`
+	}
+
 	callbackBlock := ""
 	if cfg.InstallCallback != "" {
 		callbackBlock = fmt.Sprintf(
@@ -123,21 +131,7 @@ fi
 # Set hostname
 hostnamectl set-hostname %s

-# Pre-start OVS and OVN central so their databases are initialised before
-# setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes
-# longer than setup-ovn.sh's internal 15-second timeout allows. Starting them
-# here and waiting until the NB DB is ready means setup-ovn.sh sees a live DB
-# the moment it starts — no races, no timeout failures.
-systemctl start openvswitch-switch
-systemctl start ovn-central
-echo "Waiting for OVN NB DB to initialise..."
-for _i in $(seq 1 120); do
-    if ovn-nbctl --timeout=2 get-connection >/dev/null 2>&1; then
-        echo "OVN NB DB ready (${_i}s)"
-        break
-    fi
-    sleep 1
-done
+%s

 # Create default nameservers if DHCP server returns blank
 printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" > /etc/resolvconf/resolv.conf.d/base
@@ -229,7 +223,7 @@ if grep -q 'external_mode.*pool' /etc/spinifex/spinifex.toml 2>/dev/null; then
        echo "[firstboot] warning: br-ext not up after 30s — external networking may be delayed"
    fi
 fi
-`, cfg.Hostname, setupOVN, clusterCmd, callbackBlock)
+`, cfg.Hostname, ovnPrestart, setupOVN, clusterCmd, callbackBlock)

 	path := filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh")
 	return os.WriteFile(path, []byte(script), 0o755)
@@ -66,6 +66,54 @@ func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
 	}
 }

+func TestWriteScriptRunsOVNWhenFormationOwned(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	content := readScript(t, root)
+
+	if !strings.Contains(content, "setup-ovn.sh --management") {
+		t.Error("script should run setup-ovn --management when firstboot owns formation")
+	}
+	if !strings.Contains(content, "systemctl start ovn-central") {
+		t.Error("script should pre-start ovn-central when firstboot owns formation")
+	}
+}
+
+func TestWriteScriptDefersOVNWhenSkipFormation(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node", ClusterRole: "init", SkipFormation: true}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	content := readScript(t, root)
+
+	if strings.Contains(content, "setup-ovn.sh --management") {
+		t.Error("script must not run setup-ovn --management when a controller owns OVN")
+	}
+	if strings.Contains(content, "systemctl start ovn-central") {
+		t.Error("script must not pre-start ovn-central when a controller owns OVN")
+	}
+	if !strings.Contains(content, "setup-ovn deferred") {
+		t.Error("script should note setup-ovn is deferred under SkipFormation")
+	}
+}
+
+func readScript(t *testing.T, root string) string {
+	t.Helper()
+	b, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
+	if err != nil {
+		t.Fatalf("read script: %v", err)
+	}
+	return string(b)
+}
+
 func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package install performs the disk installation steps after the UI has
 // collected configuration.
 package install
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 package install

 import (
@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package main

 import (
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program. If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package systemd writes systemd unit files and drop-ins into an installed
 // system root during the Spinifex ISO installation process.
 package systemd
@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package ui presents the interactive installer TUI using bubbletea and lipgloss.
 package ui

@@ -49,12 +49,13 @@ func main() {
 		slog.Error("--gateway is required (or set LB_GATEWAY_URL)")
 		os.Exit(1)
 	}
-	if accessKey == "" || secretKey == "" {
-		slog.Error("--access-key and --secret-key are required (or set LB_ACCESS_KEY / LB_SECRET_KEY)")
-		os.Exit(1)
+	// Empty keys select IMDS instance-role credentials (the SDK default chain
+	// falls back to the in-VM IMDS endpoint); static keys sign as-is.
+	credSource := "imds"
+	if accessKey != "" && secretKey != "" {
+		credSource = "static"
 	}
-
-	slog.Info("Starting LB agent", "lbId", lbID, "gateway", gatewayURL)
+	slog.Info("Starting LB agent", "lbId", lbID, "gateway", gatewayURL, "credSource", credSource)

 	agent, err := lbagent.New(lbID, gatewayURL, accessKey, secretKey, region)
 	if err != nil {
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd

 import (
@@ -50,6 +34,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/gpu"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/mulgadc/spinifex/spinifex/network/host"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/viperblock/viperblock"
@@ -194,6 +179,18 @@ unless --force is passed. Prompts for confirmation unless --yes is passed.`,
 	Run: runimagesRemoveCmd,
 }

+var imagesPromoteCmd = &cobra.Command{
+	Use:   "promote",
+	Short: "Promote an account-owned AMI to a system image",
+	Long: `Rewrite an account-owned AMI's owner to the system alias so it becomes
+visible to all accounts via DescribeImages, matching the behaviour of AMIs
+imported via 'spx admin images import'.
+
+No data is copied — only the config.json owner field is updated. The change
+takes effect immediately. Prompts for confirmation unless --yes is passed.`,
+	Run: runimagesPromoteCmd,
+}
+
 var accountCmd = &cobra.Command{
 	Use:   "account",
 	Short: "Manage Spinifex accounts",
@@ -285,6 +282,7 @@ func init() {
 	imagesCmd.AddCommand(imagesImportCmd)
 	imagesCmd.AddCommand(imagesListCmd)
 	imagesCmd.AddCommand(imagesRemoveCmd)
+	imagesCmd.AddCommand(imagesPromoteCmd)

 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
@@ -328,7 +326,7 @@ func init() {
 	adminInitCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all). Valid: nats,predastore,viperblock,daemon,awsgw,ui")

 	// External networking flags
-	adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (default when WAN detected) or '' (disabled)")
+	adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (default when WAN detected), 'nat' (routed; non-bridgeable uplinks; add --external-pool or --external-source=dhcp for public IPs), or '' (disabled)")
 	adminInitCmd.Flags().String("external-iface", "", "WAN NIC for br-external (auto-detected from default route)")
 	adminInitCmd.Flags().String("external-source", "", "Pool IP source: 'dhcp' (default when no --external-pool) or 'static' (uses --external-pool range)")
 	adminInitCmd.Flags().String("external-bind-bridge", "", "Linux bridge for upstream DHCP DORA (default 'br-wan' when --external-source=dhcp)")
@@ -380,6 +378,12 @@ func init() {
 	if err := imagesRemoveCmd.MarkFlagRequired("image-id"); err != nil {
 		panic(err)
 	}
+
+	imagesPromoteCmd.Flags().String("image-id", "", "AMI ID to promote to system image (required)")
+	imagesPromoteCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
+	if err := imagesPromoteCmd.MarkFlagRequired("image-id"); err != nil {
+		panic(err)
+	}
 }

 func runimagesImportCmd(cmd *cobra.Command, args []string) {
@@ -873,6 +877,67 @@ func printDependents(w io.Writer, d admin.Dependents) {
 	}
 }

+func runimagesPromoteCmd(cmd *cobra.Command, args []string) {
+	imageID, _ := cmd.Flags().GetString("image-id")
+	yes, _ := cmd.Flags().GetBool("yes")
+
+	cfgFile, _ := cmd.Flags().GetString("config")
+	if cfgFile == "" {
+		cfgFile = DefaultConfigFile()
+	}
+
+	appConfig, err := config.LoadConfig(cfgFile)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Error loading config file:", err)
+		os.Exit(1)
+	}
+
+	node := appConfig.Nodes[appConfig.Node]
+	store := objectstore.NewS3ObjectStoreFromConfig(
+		node.Predastore.Host,
+		node.Predastore.Region,
+		node.Predastore.AccessKey,
+		node.Predastore.SecretKey,
+	)
+	bucket := node.Predastore.Bucket
+
+	// Read current metadata for the confirmation prompt.
+	meta, err := admin.GetAMIMetadata(store, bucket, imageID)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Failed to inspect AMI:", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("About to promote AMI to system image:")
+	fmt.Println()
+	fmt.Printf("  Image ID:       %s\n", imageID)
+	fmt.Printf("  Name:           %s\n", meta.Name)
+	fmt.Printf("  Current owner:  %s\n", meta.ImageOwnerAlias)
+	fmt.Printf("  New owner:      %s\n", admin.SystemOwnerAlias)
+	if !meta.CreationDate.IsZero() {
+		fmt.Printf("  Created:        %s\n", meta.CreationDate.UTC().Format("2006-01-02T15:04:05Z"))
+	}
+	fmt.Println()
+	fmt.Println("After promotion this AMI will be visible to all accounts.")
+
+	if !yes {
+		fmt.Print("Type 'yes' to proceed: ")
+		reader := bufio.NewReader(os.Stdin)
+		answer, _ := reader.ReadString('\n')
+		if strings.TrimSpace(answer) != "yes" {
+			fmt.Println("Aborted.")
+			return
+		}
+	}
+
+	if _, err := admin.PromoteSystemImage(store, bucket, admin.PromoteImageOpts{ImageID: imageID}); err != nil {
+		fmt.Fprintln(os.Stderr, "Promote failed:", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("✅ Promoted %s to system image (owner: %s).\n", imageID, admin.SystemOwnerAlias)
+}
+
 // List remote images available
 func runimagesListCmd(cmd *cobra.Command, args []string) {
 	//fmt.Println(availableImages)
@@ -1026,6 +1091,13 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 				// --external-pool is omitted the validator below will error with
 				// a SuggestPoolRange hint.
 				if externalMode == "" && !cmd.Flags().Changed("external-mode") {
+					if isNonBridgeableUplink(detected.WAN.Name) {
+						fmt.Fprintf(os.Stderr, "\n❌ Detected WAN interface %s cannot be bridged (WiFi/cellular/PPP).\n", detected.WAN.Name)
+						fmt.Fprintf(os.Stderr, "   Use routed NAT mode instead (outbound-only VM networking):\n")
+						fmt.Fprintf(os.Stderr, "     ./scripts/setup-ovn.sh --management --nat-uplink\n")
+						fmt.Fprintf(os.Stderr, "     spx admin init --external-mode=nat\n")
+						os.Exit(1)
+					}
 					externalMode = "pool"
 				}
 			}
@@ -1033,60 +1105,60 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}

 	// Validate external networking flags
-	if externalMode != "" && externalMode != "pool" {
-		fmt.Fprintf(os.Stderr, "❌ Error: --external-mode must be 'pool' or empty, got: %s\n", externalMode)
+	if externalMode != "" && externalMode != "pool" && externalMode != "nat" {
+		fmt.Fprintf(os.Stderr, "❌ Error: --external-mode must be 'pool', 'nat', or empty, got: %s\n", externalMode)
 		os.Exit(1)
 	}
-	if externalMode == "pool" {
-		// Default source: dhcp when no --external-pool, else static.
-		if externalSource == "" {
-			if externalPool == "" {
-				externalSource = "dhcp"
-			} else {
-				externalSource = "static"
-			}
-		}
-		switch externalSource {
-		case "dhcp":
-			if externalPool != "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool not allowed with --external-source=dhcp (addresses come from upstream DHCP server)\n")
-				os.Exit(1)
-			}
-			if externalBindBridge == "" {
-				externalBindBridge = "br-wan"
-			}
-		case "static":
-			if externalBindBridge != "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-bind-bridge only valid with --external-source=dhcp\n")
-				os.Exit(1)
-			}
-			if externalPool == "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool is required with --external-source=static (e.g., 192.168.1.150-192.168.1.250)\n")
-				if detectedNet != nil && detectedNet.WAN != nil {
-					sugStart, sugEnd := admin.SuggestPoolRange(detectedNet.WAN)
-					fmt.Fprintf(os.Stderr, "   Suggested: --external-pool=%s-%s\n", sugStart, sugEnd)
-				}
-				os.Exit(1)
-			}
-			if externalGateway == "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is required with --external-source=static\n")
-				os.Exit(1)
-			}
-			parts := strings.SplitN(externalPool, "-", 2)
-			if len(parts) != 2 || net.ParseIP(parts[0]) == nil || net.ParseIP(parts[1]) == nil {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool must be start-end IPs (e.g., 192.168.1.150-192.168.1.250), got: %s\n", externalPool)
-				os.Exit(1)
-			}
-			poolStart, poolEnd = parts[0], parts[1]
-		default:
-			fmt.Fprintf(os.Stderr, "❌ Error: --external-source must be 'static' or 'dhcp', got: %s\n", externalSource)
+	// A public pool alongside nat's transit pool restores EIP / public-subnet
+	// parity where the operator has spare LAN IPs; without these flags nat
+	// stays Tier-1-only (host-jumpbox access, no public IPs).
+	natPublicPool := externalMode == "nat" && (externalPool != "" || externalSource != "")
+	// natPublicGateway keeps the upstream gateway for the public pool before
+	// the transit segment claims externalGateway below.
+	natPublicGateway := externalGateway
+	if externalMode == "nat" {
+		if nodes >= 2 {
+			fmt.Fprintf(os.Stderr, "❌ Error: --external-mode=nat is single-node only (v1); use --nodes=1\n")
 			os.Exit(1)
 		}
+		if !natPublicPool && (externalBindBridge != "" || gatewayIP != "") {
+			fmt.Fprintf(os.Stderr, "❌ Error: --external-bind-bridge/--gateway-ip require --external-pool or --external-source in --external-mode=nat\n")
+			os.Exit(1)
+		}
+		if natPublicPool {
+			// DHCP DORA in nat mode binds the uplink interface itself — there
+			// is no br-wan (nothing is bridged in routed mode).
+			src, start, end, bb, err := resolvePublicPoolFlags(externalSource, externalPool, externalBindBridge, natPublicGateway, externalIface)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+				os.Exit(1)
+			}
+			externalSource, poolStart, poolEnd, externalBindBridge = src, start, end, bb
+		}
+		// The transit segment is fixed: the host veth owns the gateway IP and
+		// masquerades the /24.
+		externalGateway = host.NATTransitGatewayIP
+	}
+	if externalMode == "pool" {
+		src, start, end, bb, err := resolvePublicPoolFlags(externalSource, externalPool, externalBindBridge, externalGateway, "br-wan")
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+			if strings.Contains(err.Error(), "--external-pool is required") && detectedNet != nil && detectedNet.WAN != nil {
+				sugStart, sugEnd := admin.SuggestPoolRange(detectedNet.WAN)
+				fmt.Fprintf(os.Stderr, "   Suggested: --external-pool=%s-%s\n", sugStart, sugEnd)
+			}
+			os.Exit(1)
+		}
+		externalSource, poolStart, poolEnd, externalBindBridge = src, start, end, bb
 	}
 	if externalGateway != "" && net.ParseIP(externalGateway) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is not a valid IP: %s\n", externalGateway)
 		os.Exit(1)
 	}
+	if natPublicGateway != "" && net.ParseIP(natPublicGateway) == nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is not a valid IP: %s\n", natPublicGateway)
+		os.Exit(1)
+	}
 	if gatewayIP != "" && net.ParseIP(gatewayIP) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: --gateway-ip is not a valid IP: %s\n", gatewayIP)
 		os.Exit(1)
@@ -1101,6 +1173,36 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		}
 	}

+	// Assemble the external pool blocks rendered into spinifex.toml.
+	var externalPools []admin.PoolData
+	switch externalMode {
+	case "nat":
+		externalPools = append(externalPools, admin.PoolData{
+			Name: host.NATTransitPoolName, Gateway: host.NATTransitGatewayIP,
+			PrefixLen: 24, DNSServers: dnsServers,
+			GwLrpRangeStart: host.NATTransitGwLrpStart, GwLrpRangeEnd: host.NATTransitGwLrpEnd,
+		})
+		if natPublicPool {
+			// WiFi/WWAN uplinks drop frames with foreign source MACs, so DHCP
+			// leases must go out with the interface's own MAC.
+			dhcpMAC := ""
+			if externalSource == "dhcp" && isNonBridgeableUplink(externalBindBridge) {
+				dhcpMAC = "interface"
+			}
+			externalPools = append(externalPools, admin.PoolData{
+				Name: "wan", Source: externalSource, BindBridge: externalBindBridge,
+				DHCPMAC: dhcpMAC, Start: poolStart, End: poolEnd, Gateway: natPublicGateway,
+				GatewayIP: gatewayIP, PrefixLen: externalPrefixLen, DNSServers: dnsServers,
+			})
+		}
+	case "pool":
+		externalPools = append(externalPools, admin.PoolData{
+			Name: "wan", Source: externalSource, BindBridge: externalBindBridge,
+			Start: poolStart, End: poolEnd, Gateway: externalGateway,
+			GatewayIP: gatewayIP, PrefixLen: externalPrefixLen, DNSServers: dnsServers,
+		})
+	}
+
 	// Validate IP address format
 	if net.ParseIP(bindIP) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP address for --bind: %s\n", bindIP)
@@ -1165,66 +1267,91 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}
 	fmt.Printf("✅ Created config directory: %s\n", configDir)

-	// Generate system credentials (for service-to-service auth in config files)
-	accessKey, err := admin.GenerateAWSAccessKey()
+	// Identity and crypto material is load-or-generate: a fresh install mints a
+	// new identity bundle, but a --force re-init preserves the existing one so
+	// data sealed under it (NATS KV secrets, sealed fragments, encrypted volumes)
+	// stays decryptable.
+	masterKey, masterKeyExisted, err := ensureMasterKey(configDir)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error generating access key: %v\n", err)
-		os.Exit(1)
-	}
-	secretKey, err := admin.GenerateAWSSecretKey()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error generating secret key: %v\n", err)
+		fmt.Fprintf(os.Stderr, "Error preparing IAM master key: %v\n", err)
 		os.Exit(1)
 	}
 	accountID := admin.SystemAccountID()
-
-	// Generate IAM master key (AES-256, used to encrypt secrets in NATS KV)
-	masterKey, err := handlers_iam.GenerateMasterKey()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error generating IAM master key: %v\n", err)
-		os.Exit(1)
-	}
 	bootstrapDir := filepath.Join(spxRoot, "awsgw")
-	bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error writing bootstrap files: %v\n", err)
-		os.Exit(1)
-	}
-	if err := writeSystemCredentials(configDir, accessKey, secretKey); err != nil {
-		fmt.Fprintf(os.Stderr, "Error writing system credentials: %v\n", err)
-		os.Exit(1)
-	}
-	fmt.Println("\n🔐 Generated IAM master key")
-	fmt.Printf("   Master key: %s\n", filepath.Join(configDir, "master.key"))
-	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
-	fmt.Printf("   System creds: %s\n", filepath.Join(configDir, "system-credentials.json"))

-	// Predastore encryption key is per-node and never transmitted; generate
-	// it locally now so the service has it on first start.
+	var accessKey, secretKey, adminAccessKey, adminSecretKey string
+	if masterKeyExisted {
+		// Preserve path: reuse the existing identity. The system credentials must
+		// match what seeded the NATS KV `system` secret, so load them rather than
+		// mint new ones. The admin credentials are not recovered: bootstrap.json is
+		// consumed and deleted by awsgw after first boot, and the operator's copy
+		// already lives in ~/.aws/credentials. Leaving them empty makes
+		// finalizeNodeSetup refresh only ~/.aws/config (endpoint/CA for a changed
+		// bind IP), not the credentials.
+		accessKey, secretKey, err = loadSystemCredentials(configDir)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error loading preserved system credentials: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("\n🔐 Preserved existing identity (master key, credentials and CA unchanged)")
+		fmt.Printf("   Master key: %s\n", filepath.Join(configDir, "master.key"))
+	} else {
+		// Fresh install: mint system + admin credentials and seed the bootstrap files.
+		accessKey, err = admin.GenerateAWSAccessKey()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error generating access key: %v\n", err)
+			os.Exit(1)
+		}
+		secretKey, err = admin.GenerateAWSSecretKey()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error generating secret key: %v\n", err)
+			os.Exit(1)
+		}
+		bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error writing bootstrap files: %v\n", err)
+			os.Exit(1)
+		}
+		if err := writeSystemCredentials(configDir, accessKey, secretKey); err != nil {
+			fmt.Fprintf(os.Stderr, "Error writing system credentials: %v\n", err)
+			os.Exit(1)
+		}
+		adminAccessKey = bootstrapResult.AdminAccessKey
+		adminSecretKey = bootstrapResult.AdminSecretKey
+		fmt.Println("\n🔐 Generated IAM master key")
+		fmt.Printf("   Master key: %s\n", filepath.Join(configDir, "master.key"))
+		fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
+		fmt.Printf("   System creds: %s\n", filepath.Join(configDir, "system-credentials.json"))
+	}
+
+	// Predastore encryption key is per-node and never transmitted; load-or-generate
+	// so the service has it on first start and a re-init keeps sealed fragments.
 	predastoreKeyPath, err := writePredastoreEncryptionKey(configDir)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error generating predastore encryption key: %v\n", err)
+		fmt.Fprintf(os.Stderr, "Error preparing predastore encryption key: %v\n", err)
 		os.Exit(1)
 	}
-	fmt.Println("\n🔐 Generated predastore encryption key (per-node, never transmitted)")
+	fmt.Println("\n🔐 Predastore encryption key ready (per-node, never transmitted)")
 	fmt.Printf("   Key: %s\n", predastoreKeyPath)

-	// Viperblock at-rest encryption key is cluster-wide; on a single-node init
-	// there are no joiners, so generate it locally and enable encryption by
-	// default for all volumes created on this install.
-	viperblockKeyPath, err := writeViperblockEncryptionKey(configDir)
+	// Viperblock at-rest encryption key is cluster-wide; load-or-generate so a
+	// re-init keeps the existing key (and its sealed volumes). The bytes feed the
+	// multi-node leader's key distribution below.
+	viperblockKey, viperblockKeyPath, err := ensureViperblockEncryptionKey(configDir)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error generating viperblock encryption key: %v\n", err)
+		fmt.Fprintf(os.Stderr, "Error preparing viperblock encryption key: %v\n", err)
 		os.Exit(1)
 	}
-	fmt.Println("\n🔐 Generated viperblock at-rest encryption key")
+	fmt.Println("\n🔐 Viperblock at-rest encryption key ready")
 	fmt.Printf("   Key: %s\n", viperblockKeyPath)

-	fmt.Printf("\n🔑 Generated admin credentials (save these — they won't be shown again):\n")
-	fmt.Printf("   Access Key:  %s\n", bootstrapResult.AdminAccessKey)
-	fmt.Printf("   Secret Key:  %s\n", bootstrapResult.AdminSecretKey)
-	fmt.Printf("   Account:     %s (%s)\n", admin.DefaultAccountName(), admin.DefaultAccountID())
-	fmt.Printf("   AWS Profile: spinifex\n")
+	if !masterKeyExisted {
+		fmt.Printf("\n🔑 Generated admin credentials (save these — they won't be shown again):\n")
+		fmt.Printf("   Access Key:  %s\n", adminAccessKey)
+		fmt.Printf("   Secret Key:  %s\n", adminSecretKey)
+		fmt.Printf("   Account:     %s (%s)\n", admin.DefaultAccountName(), admin.DefaultAccountID())
+		fmt.Printf("   AWS Profile: spinifex\n")
+	}

 	// Generate SSL certificates (with bind IP in SANs for multi-node support)
 	certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP, region, config.DefaultAWSInternalSuffix)
@@ -1290,7 +1417,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 			networkConfig.BootstrapSubnetCidr = handlers_ec2_vpc.DefaultSubnetCidr
 		}

-		runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, natsToken, clusterName,
+		runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey,
+			masterKey, viperblockKey, natsToken, clusterName,
 			configDir, spxRoot, certPath, region, az, node, bindIP, advertiseIP, clusterBind, email,
 			port, nodes, formationTimeoutStr, tokenTTLStr, services, networkConfig)
 		return
@@ -1386,17 +1514,10 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		OVNNBAddr: "tcp:127.0.0.1:6641",
 		OVNSBAddr: "tcp:127.0.0.1:6642",

-		ExternalMode:   externalMode,
-		ExternalIface:  externalIface,
-		PoolName:       "wan",
-		PoolSource:     externalSource,
-		PoolBindBridge: externalBindBridge,
-		PoolStart:      poolStart,
-		PoolEnd:        poolEnd,
-		PoolGateway:    externalGateway,
-		PoolGatewayIP:  gatewayIP,
-		PoolPrefixLen:  externalPrefixLen,
-		PoolDNSServers: dnsServers,
+		ExternalMode:  externalMode,
+		ExternalIface: externalIface,
+		BridgeMode:    bridgeModeFor(externalMode),
+		Pools:         externalPools,

 		OperatorEmail:       email,
 		BootstrapAccountId:  admin.DefaultAccountID(),
@@ -1413,7 +1534,20 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}

 	// Print external networking summary
-	if externalMode != "" {
+	if externalMode == "nat" {
+		if natPublicPool {
+			fmt.Printf("\n📡 External networking: nat (routed) with public pool — EIPs enabled\n")
+			if externalSource == "static" {
+				fmt.Printf("  Public pool:   %s - %s (source: static)\n", poolStart, poolEnd)
+			} else {
+				fmt.Printf("  Public pool:   dhcp via %s\n", externalBindBridge)
+			}
+		} else {
+			fmt.Printf("\n📡 External networking: nat (routed, outbound-only — no public IPs/EIPs)\n")
+		}
+		fmt.Printf("  Transit:       %s via %s (host masquerades out any uplink)\n", host.NATTransitCIDR, host.NATTransitHostEnd)
+		fmt.Printf("  Host setup:    ./scripts/setup-ovn.sh --nat-uplink (run before starting services)\n")
+	} else if externalMode != "" {
 		fmt.Printf("\n📡 External networking: %s\n", externalMode)
 		fmt.Printf("  WAN interface: %s\n", externalIface)
 		switch externalSource {
@@ -1436,7 +1570,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}

-	finalizeNodeSetup(spxRoot, certPath, bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, bindIP, advertiseIP)
+	finalizeNodeSetup(spxRoot, certPath, adminAccessKey, adminSecretKey, region, bindIP)

 	// Write node.conf so spx admin banner works on source installs (not just ISO).
 	nodeHostname, _ := os.Hostname()
@@ -1467,7 +1601,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 // runAdminInitMultiNode handles the multi-node formation path for admin init.
 // It starts a formation server, registers this node, waits for all nodes to join,
 // then generates configs with complete cluster topology.
-func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, natsToken, clusterName,
+func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID, adminAccessKey, adminSecretKey string,
+	masterKey, viperblockKey []byte, natsToken, clusterName,
 	configDir, spxRoot, certPath, region, az, node, bindIP, advertiseIP, clusterBind, email string,
 	port, expectedNodes int, formationTimeoutStr, tokenTTLStr string, services []string, networkConfig *formation.NetworkConfig) {
 	formationTimeout, err := time.ParseDuration(formationTimeoutStr)
@@ -1494,47 +1629,11 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}

-	// Generate IAM master key for the cluster
-	masterKey, err := handlers_iam.GenerateMasterKey()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error generating IAM master key: %v\n", err)
-		os.Exit(1)
-	}
-	bootstrapDir := filepath.Join(spxRoot, "awsgw")
-	bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error writing bootstrap files: %v\n", err)
-		os.Exit(1)
-	}
-	fmt.Println("\n🔐 Generated IAM master key")
-	fmt.Printf("   Bootstrap: %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
-
-	// Predastore encryption key is per-node and never distributed via the
-	// formation server. Generate only the leader's own key here; each
-	// joiner generates its own during `spx admin join`.
-	predastoreKeyPath, err := writePredastoreEncryptionKey(configDir)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error generating predastore encryption key: %v\n", err)
-		os.Exit(1)
-	}
-	fmt.Println("\n🔐 Generated predastore encryption key (per-node, never transmitted)")
-	fmt.Printf("   Key: %s\n", predastoreKeyPath)
-
-	// Viperblock at-rest encryption key is cluster-wide and shared: the leader
-	// generates it once and distributes it to joiners via the formation server
-	// so a volume sealed on any node can be opened on any other.
-	viperblockKey, err := handlers_iam.GenerateMasterKey()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error generating viperblock encryption key: %v\n", err)
-		os.Exit(1)
-	}
-	viperblockKeyPath, err := saveViperblockEncryptionKey(configDir, viperblockKey)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "❌ Error saving viperblock encryption key: %v\n", err)
-		os.Exit(1)
-	}
-	fmt.Println("\n🔐 Generated viperblock at-rest encryption key (cluster-wide, shared with joiners)")
-	fmt.Printf("   Key: %s\n", viperblockKeyPath)
+	// Identity, predastore and viperblock keys were prepared load-or-generate by
+	// the caller: reuse them here so a re-init never rotates cluster crypto and
+	// the leader distributes the same viperblock key to joiners. The path is
+	// deterministic and already written on disk.
+	viperblockKeyPath := filepath.Join(configDir, "viperblock", "encryption.key")

 	// Read CA cert/key for distribution to joining nodes
 	caCertData, err := os.ReadFile(filepath.Join(configDir, "ca.pem"))
@@ -1555,8 +1654,8 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		NatsToken:      natsToken,
 		ClusterName:    clusterName,
 		Region:         region,
-		AdminAccessKey: bootstrapResult.AdminAccessKey,
-		AdminSecretKey: bootstrapResult.AdminSecretKey,
+		AdminAccessKey: adminAccessKey,
+		AdminSecretKey: adminSecretKey,
 	}

 	fs := formation.NewFormationServer(expectedNodes, creds, string(caCertData), string(caKeyData), networkConfig, joinToken, tokenTTL)
@@ -1616,6 +1715,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	allNodes := fs.Nodes()
 	clusterRoutes := formation.BuildClusterRoutes(allNodes)
 	predastoreNodes := formation.BuildPredastoreNodes(allNodes)
+	ovnNBAddr, ovnSBAddr := formation.BuildOVNDBAddrs(allNodes)

 	fmt.Println("\n📝 Creating configuration files...")

@@ -1677,9 +1777,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,

 		EncryptionKeyFile: viperblockKeyPath,

-		// Init node runs ovn-central locally
-		OVNNBAddr: "tcp:127.0.0.1:6641",
-		OVNSBAddr: "tcp:127.0.0.1:6642",
+		// Multi-endpoint OVN NB/SB list across the RAFT quorum; the init node's
+		// own address leads, the rest provide failover.
+		OVNNBAddr: ovnNBAddr,
+		OVNSBAddr: ovnSBAddr,
 	}

 	if networkConfig != nil {
@@ -1691,7 +1792,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}

-	finalizeNodeSetup(spxRoot, certPath, bootstrapResult.AdminAccessKey, bootstrapResult.AdminSecretKey, region, bindIP, advertiseIP)
+	finalizeNodeSetup(spxRoot, certPath, adminAccessKey, adminSecretKey, region, bindIP)

 	// Keep formation server running briefly so joining nodes can fetch complete status
 	fmt.Println("\n⏳ Waiting for joining nodes to fetch cluster data...")
@@ -1764,13 +1865,6 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}

-	// Extract leader IP for OVN NB/SB DB address (strip port from host:port)
-	leaderIP, _, err := net.SplitHostPort(leaderHost)
-	if err != nil {
-		// leaderHost might be an IP without port
-		leaderIP = leaderHost
-	}
-
 	// Validate IP address format
 	if net.ParseIP(bindIP) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP address for --bind: %s\n", bindIP)
@@ -2073,6 +2167,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	// Build cluster topology from formation data
 	clusterRoutes := formation.BuildClusterRoutes(statusResp.Nodes)
 	predastoreNodes := formation.BuildPredastoreNodes(statusResp.Nodes)
+	ovnNBAddr, ovnSBAddr := formation.BuildOVNDBAddrs(statusResp.Nodes)

 	fmt.Println("📝 Creating configuration files...")

@@ -2139,9 +2234,10 @@ func runAdminJoin(cmd *cobra.Command, args []string) {

 		EncryptionKeyFile: viperblockKeyPath,

-		// Joining nodes connect to the init node's OVN NB/SB DB
-		OVNNBAddr: fmt.Sprintf("tcp:%s:6641", leaderIP),
-		OVNSBAddr: fmt.Sprintf("tcp:%s:6642", leaderIP),
+		// Multi-endpoint OVN NB/SB list across the RAFT quorum so the client
+		// fails over instead of pinning to a single init node.
+		OVNNBAddr: ovnNBAddr,
+		OVNSBAddr: ovnSBAddr,
 	}

 	if statusResp.NetworkConfig != nil {
@@ -2153,7 +2249,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}

-	finalizeNodeSetup(dataDir, caCertPath, creds.AdminAccessKey, creds.AdminSecretKey, creds.Region, bindIP, advertiseIP)
+	finalizeNodeSetup(dataDir, caCertPath, creds.AdminAccessKey, creds.AdminSecretKey, creds.Region, bindIP)

 	// Print cluster summary
 	fmt.Println("\n🎉 Node successfully joined cluster!")
@@ -2410,7 +2506,7 @@ func runCertRenew(cmd *cobra.Command, _ []string) {
 	serverCertPath := filepath.Join(configDir, "server.pem")
 	serverKeyPath := filepath.Join(configDir, "server.key")

-	if err := admin.GenerateSignedCertWithDNS(serverCertPath, serverKeyPath, caCertPath, caKeyPath, extraIPs, extraDNS); err != nil {
+	if err := admin.GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, extraIPs, extraDNS); err != nil {
 		fmt.Fprintf(os.Stderr, "Error regenerating server certificate: %v\n", err)
 		os.Exit(1)
 	}
@@ -2463,12 +2559,10 @@ func generateAndWriteConfigs(dirs configDirs, spinifexTomlPath string, settings
 }

 // finalizeNodeSetup configures AWS credentials, creates service directories,
-// and sets ownership when running as root. advertiseIP is the off-host dial
-// target for this node; it is threaded through SetupAWSCredentials's wanIP
-// param for a future --operator-endpoint flag (see SetupAWSCredentials doc).
-func finalizeNodeSetup(dataDir, certPath, adminAccessKey, adminSecretKey, region, bindIP, advertiseIP string) {
+// and sets ownership when running as root.
+func finalizeNodeSetup(dataDir, certPath, adminAccessKey, adminSecretKey, region, bindIP string) {
 	fmt.Println("\n🔧 Configuring AWS credentials...")
-	if err := admin.SetupAWSCredentials(adminAccessKey, adminSecretKey, region, certPath, bindIP, advertiseIP); err != nil {
+	if err := admin.SetupAWSCredentials(adminAccessKey, adminSecretKey, region, certPath, bindIP); err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: Could not update AWS credentials: %v\n", err)
 	} else {
 		fmt.Println("✅ AWS credentials configured")
@@ -2486,15 +2580,19 @@ func finalizeNodeSetup(dataDir, certPath, adminAccessKey, adminSecretKey, region
 func applyNetworkConfig(settings *admin.ConfigSettings, nc *formation.NetworkConfig) {
 	settings.IPSecEnabled = nc.IPSecEnabled
 	settings.ExternalMode = nc.ExternalMode
-	settings.PoolName = nc.PoolName
-	settings.PoolSource = nc.PoolSource
-	settings.PoolBindBridge = nc.PoolBindBridge
-	settings.PoolStart = nc.PoolStart
-	settings.PoolEnd = nc.PoolEnd
-	settings.PoolGateway = nc.PoolGateway
-	settings.PoolGatewayIP = nc.PoolGatewayIP
-	settings.PoolPrefixLen = nc.PoolPrefixLen
-	settings.PoolDNSServers = nc.PoolDNSServers
+	if nc.ExternalMode != "" {
+		settings.Pools = []admin.PoolData{{
+			Name:       nc.PoolName,
+			Source:     nc.PoolSource,
+			BindBridge: nc.PoolBindBridge,
+			Start:      nc.PoolStart,
+			End:        nc.PoolEnd,
+			Gateway:    nc.PoolGateway,
+			GatewayIP:  nc.PoolGatewayIP,
+			PrefixLen:  nc.PoolPrefixLen,
+			DNSServers: nc.PoolDNSServers,
+		}}
+	}

 	settings.BootstrapAccountId = nc.BootstrapAccountId
 	settings.BootstrapVpcId = nc.BootstrapVpcId
@@ -2518,6 +2616,48 @@ type writeBootstrapResult struct {
 	AdminSecretKey string
 }

+// ensureMasterKey load-or-generates the IAM master key at <configDir>/master.key.
+// It returns the key bytes and whether the key already existed on disk. On a
+// --force re-init the existing key is preserved: rotating it would orphan every
+// IAM secret in NATS KV (e.g. the ECR signing key) that was encrypted under it.
+func ensureMasterKey(configDir string) (key []byte, existed bool, err error) {
+	keyPath := filepath.Join(configDir, "master.key")
+	if admin.FileExists(keyPath) {
+		key, err = handlers_iam.LoadMasterKey(keyPath)
+		if err != nil {
+			return nil, false, fmt.Errorf("load master key: %w", err)
+		}
+		return key, true, nil
+	}
+	key, err = handlers_iam.GenerateMasterKey()
+	if err != nil {
+		return nil, false, fmt.Errorf("generate master key: %w", err)
+	}
+	return key, false, nil
+}
+
+// loadSystemCredentials reads the preserved system access/secret key from
+// <configDir>/system-credentials.json. On a re-init the configs must render the
+// same system credentials that seeded the NATS KV `system` secret, else SigV4
+// auth between services fails.
+func loadSystemCredentials(configDir string) (accessKey, secretKey string, err error) {
+	data, err := os.ReadFile(filepath.Join(configDir, "system-credentials.json"))
+	if err != nil {
+		return "", "", fmt.Errorf("read system credentials: %w", err)
+	}
+	var creds struct {
+		AccessKeyID     string `json:"access_key_id"`
+		SecretAccessKey string `json:"secret_access_key"`
+	}
+	if err := json.Unmarshal(data, &creds); err != nil {
+		return "", "", fmt.Errorf("parse system credentials: %w", err)
+	}
+	if creds.AccessKeyID == "" || creds.SecretAccessKey == "" {
+		return "", "", fmt.Errorf("system credentials missing access/secret key")
+	}
+	return creds.AccessKeyID, creds.SecretAccessKey, nil
+}
+
 // writeBootstrapFiles generates new admin credentials and writes the bootstrap
 // files (master.key to configDir, bootstrap.json to bootstrapDir).
 // Used by init flows (single and multi-node).
@@ -2539,47 +2679,57 @@ func writeBootstrapFiles(configDir, bootstrapDir string, masterKey []byte, acces
 	}, nil
 }

-// writePredastoreEncryptionKey generates a fresh 32-byte AES-256 master key
-// for this node's predastore data dir and writes it to
-// <configDir>/predastore/encryption.key with mode 0600. The key is per-node:
-// every node generates its own at init/join time and never transmits it,
-// because predastore only opens fragments on the node that sealed them.
+// writePredastoreEncryptionKey load-or-generates this node's per-node predastore
+// key at <configDir>/predastore/encryption.key (mode 0600). The key is per-node:
+// every node generates its own at init/join time and never transmits it, because
+// predastore only opens fragments on the node that sealed them. An existing key
+// is preserved so a --force re-init does not orphan already-sealed fragments.
 func writePredastoreEncryptionKey(configDir string) (string, error) {
 	predastoreDir := filepath.Join(configDir, "predastore")
 	if err := os.MkdirAll(predastoreDir, 0750); err != nil {
 		return "", fmt.Errorf("create predastore config dir: %w", err)
 	}
+	keyPath := filepath.Join(predastoreDir, "encryption.key")
+	if admin.FileExists(keyPath) {
+		return keyPath, nil
+	}
 	key, err := handlers_iam.GenerateMasterKey()
 	if err != nil {
 		return "", fmt.Errorf("generate predastore encryption key: %w", err)
 	}
-	keyPath := filepath.Join(predastoreDir, "encryption.key")
 	if err := handlers_iam.SaveMasterKey(keyPath, key); err != nil {
 		return "", fmt.Errorf("save predastore encryption key: %w", err)
 	}
 	return keyPath, nil
 }

-// writeViperblockEncryptionKey generates a fresh 32-byte AES-256 master key for
-// viperblock at-rest encryption and writes it to
-// <configDir>/viperblock/encryption.key with mode 0600. Unlike the predastore
-// key, this key is cluster-wide and shared: the leader generates it once at init
-// and distributes it to joiners via the formation server, so a volume sealed on
-// any node can be opened on any other. Loaded at startup via masterkey.LoadShared.
-func writeViperblockEncryptionKey(configDir string) (string, error) {
+// ensureViperblockEncryptionKey load-or-generates the cluster-wide viperblock
+// at-rest key at <configDir>/viperblock/encryption.key (mode 0600) and returns
+// its bytes and path. Unlike the predastore key it is shared: the leader loads
+// these bytes to distribute the same key to joiners via the formation server, so
+// a volume sealed on any node can be opened on any other. An existing key is
+// preserved so a --force re-init does not orphan already-encrypted volumes.
+func ensureViperblockEncryptionKey(configDir string) ([]byte, string, error) {
 	viperblockDir := filepath.Join(configDir, "viperblock")
 	if err := os.MkdirAll(viperblockDir, 0750); err != nil {
-		return "", fmt.Errorf("create viperblock config dir: %w", err)
+		return nil, "", fmt.Errorf("create viperblock config dir: %w", err)
+	}
+	keyPath := filepath.Join(viperblockDir, "encryption.key")
+	if admin.FileExists(keyPath) {
+		key, err := handlers_iam.LoadMasterKey(keyPath)
+		if err != nil {
+			return nil, "", fmt.Errorf("load viperblock encryption key: %w", err)
+		}
+		return key, keyPath, nil
 	}
 	key, err := handlers_iam.GenerateMasterKey()
 	if err != nil {
-		return "", fmt.Errorf("generate viperblock encryption key: %w", err)
+		return nil, "", fmt.Errorf("generate viperblock encryption key: %w", err)
 	}
-	keyPath := filepath.Join(viperblockDir, "encryption.key")
 	if err := handlers_iam.SaveMasterKey(keyPath, key); err != nil {
-		return "", fmt.Errorf("save viperblock encryption key: %w", err)
+		return nil, "", fmt.Errorf("save viperblock encryption key: %w", err)
 	}
-	return keyPath, nil
+	return key, keyPath, nil
 }

 // saveViperblockEncryptionKey writes an already-generated 32-byte viperblock
@@ -2652,6 +2802,66 @@ func writeBootstrapFilesWithAdmin(configDir, bootstrapDir string, masterKey []by
 	return handlers_iam.SaveBootstrapData(filepath.Join(bootstrapDir, "bootstrap.json"), bd)
 }

+// isNonBridgeableUplink reports whether the interface name indicates an uplink
+// that cannot be enslaved to an L2 bridge: WiFi (wl*), cellular (ww*), PPP.
+func isNonBridgeableUplink(name string) bool {
+	return strings.HasPrefix(name, "wl") || strings.HasPrefix(name, "ww") || strings.HasPrefix(name, "ppp")
+}
+
+// resolvePublicPoolFlags validates the public-pool flag set shared by pool
+// mode and nat-with-public-pool. Returns the resolved source, static range
+// start/end, and bind bridge. Source defaults to dhcp when no range is given;
+// defaultBindBridge fills --external-bind-bridge for dhcp (br-wan in pool
+// mode, the uplink interface in nat mode).
+func resolvePublicPoolFlags(source, poolRange, bindBridge, gateway, defaultBindBridge string) (resolvedSource, start, end, resolvedBindBridge string, err error) {
+	if source == "" {
+		if poolRange == "" {
+			source = "dhcp"
+		} else {
+			source = "static"
+		}
+	}
+	switch source {
+	case "dhcp":
+		if poolRange != "" {
+			return "", "", "", "", fmt.Errorf("--external-pool not allowed with --external-source=dhcp (addresses come from upstream DHCP server)")
+		}
+		if bindBridge == "" {
+			if defaultBindBridge == "" {
+				return "", "", "", "", fmt.Errorf("--external-bind-bridge is required with --external-source=dhcp (no uplink interface detected)")
+			}
+			bindBridge = defaultBindBridge
+		}
+	case "static":
+		if bindBridge != "" {
+			return "", "", "", "", fmt.Errorf("--external-bind-bridge only valid with --external-source=dhcp")
+		}
+		if poolRange == "" {
+			return "", "", "", "", fmt.Errorf("--external-pool is required with --external-source=static (e.g., 192.168.1.150-192.168.1.250)")
+		}
+		if gateway == "" {
+			return "", "", "", "", fmt.Errorf("--external-gateway is required with --external-source=static")
+		}
+		parts := strings.SplitN(poolRange, "-", 2)
+		if len(parts) != 2 || net.ParseIP(parts[0]) == nil || net.ParseIP(parts[1]) == nil {
+			return "", "", "", "", fmt.Errorf("--external-pool must be start-end IPs (e.g., 192.168.1.150-192.168.1.250), got: %s", poolRange)
+		}
+		start, end = parts[0], parts[1]
+	default:
+		return "", "", "", "", fmt.Errorf("--external-source must be 'static' or 'dhcp', got: %s", source)
+	}
+	return source, start, end, bindBridge, nil
+}
+
+// bridgeModeFor returns the vpcd bridge_mode admin init persists for the
+// external mode: only nat mode is pinned; bridged modes stay auto-detected.
+func bridgeModeFor(externalMode string) string {
+	if externalMode == "nat" {
+		return "nat"
+	}
+	return ""
+}
+
 // detectDNSServers auto-detects DNS servers from the host for the specified
 // interface. Uses resolvectl (systemd-resolved) first, then falls back to
 // /etc/resolv.conf. Returns up to 3 servers. Falls back to public DNS if none found.
@@ -0,0 +1,77 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
+	"github.com/spf13/cobra"
+)
+
+var adminEksCmd = &cobra.Command{
+	Use:   "eks",
+	Short: "Manage EKS control-plane disaster recovery",
+}
+
+var adminEksRestoreSnapshotCmd = &cobra.Command{
+	Use:   "restore-snapshot",
+	Short: "Rebuild a single-CP cluster's control plane from an etcd snapshot",
+	Long: `restore-snapshot drives the single-CP total-loss DR path: it launches a fresh
+control-plane VM as a cluster-init seed, directs its boot-time recovery agent to
+run k3s server --cluster-reset (restoring the given etcd snapshot, or the latest
+one found in predastore when --snapshot is omitted), and re-points the cluster
+NLB target groups at the new CP.
+
+Only single-CP clusters are supported. An HA cluster has a potentially
+surviving etcd quorum and should be recovered via quorum reformation instead
+of this destructive single-node rebuild.`,
+	Run: runAdminEksRestoreSnapshot,
+}
+
+func init() {
+	adminCmd.AddCommand(adminEksCmd)
+	adminEksCmd.AddCommand(adminEksRestoreSnapshotCmd)
+
+	adminEksRestoreSnapshotCmd.Flags().String("cluster", "", "Cluster name to restore (required)")
+	adminEksRestoreSnapshotCmd.Flags().String("snapshot", "", "Etcd snapshot key to restore; defaults to the latest snapshot found in predastore")
+	adminEksRestoreSnapshotCmd.Flags().String("account", "", "Account ID that owns the cluster; defaults to the bootstrap account")
+	_ = adminEksRestoreSnapshotCmd.MarkFlagRequired("cluster")
+}
+
+func runAdminEksRestoreSnapshot(cmd *cobra.Command, _ []string) {
+	clusterName, _ := cmd.Flags().GetString("cluster")
+	snapshot, _ := cmd.Flags().GetString("snapshot")
+	accountID, _ := cmd.Flags().GetString("account")
+	if accountID == "" {
+		accountID = admin.DefaultAccountID()
+	}
+
+	_, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	svc := handlers_eks.NewNATSEKSService(nc)
+	// The server side launches a VM (30-60s) then fences the old CP with retries,
+	// so the RPC needs generous headroom over a bare launch.
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second)
+	defer cancel()
+
+	fmt.Printf("Restoring control plane for cluster %q (account %s)...\n", clusterName, accountID)
+	out, err := svc.RestoreSnapshot(ctx, &handlers_eks.RestoreSnapshotInput{
+		ClusterName: clusterName,
+		Snapshot:    snapshot,
+	}, accountID)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("New control plane %s launched (snapshot %s)\n", out.NewInstanceID, out.Snapshot)
+	fmt.Printf("Status: %s\n", out.Status)
+}
@@ -0,0 +1,247 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSpinifexTomlTemplate_NATMode(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:      "node1",
+		Az:        "ap-southeast-2a",
+		Port:      "4432",
+		Region:    "ap-southeast-2",
+		BindIP:    "10.11.12.1",
+		AccessKey: "AKIATEST",
+		SecretKey: "SECRET",
+		AccountID: "123456789012",
+		NatsToken: "token",
+		ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "nat",
+		BridgeMode:   "nat",
+		Pools: []admin.PoolData{{
+			Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24,
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `external_mode = "nat"`)
+	assert.Contains(t, content, `bridge_mode = "nat"`)
+	assert.Contains(t, content, `name        = "nat-transit"`)
+	assert.Contains(t, content, `gateway     = "100.127.0.1"`)
+	assert.NotContains(t, content, "range_start", "nat mode has no public IP range")
+}
+
+func TestSpinifexTomlTemplate_GwLrpRange(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node: "node1", Az: "ap-southeast-2a", Port: "4432",
+		Region: "ap-southeast-2", BindIP: "10.11.12.1",
+		AccessKey: "AKIATEST", SecretKey: "SECRET", AccountID: "123456789012",
+		NatsToken: "token", ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641", OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "nat", BridgeMode: "nat",
+		Pools: []admin.PoolData{{
+			Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24,
+			GwLrpRangeStart: "100.127.0.16", GwLrpRangeEnd: "100.127.0.254",
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `gw_lrp_range_start = "100.127.0.16"`)
+	assert.Contains(t, content, `gw_lrp_range_end   = "100.127.0.254"`)
+}
+
+func TestSpinifexTomlTemplate_GwLrpRangeOmittedWhenUnset(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node: "node1", Az: "ap-southeast-2a", Port: "4432",
+		Region: "ap-southeast-2", BindIP: "10.11.12.1",
+		AccessKey: "AKIATEST", SecretKey: "SECRET", AccountID: "123456789012",
+		NatsToken: "token", ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641", OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "pool", ExternalIface: "enp0s3",
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "static",
+			Start: "192.168.1.150", End: "192.168.1.250",
+			Gateway: "192.168.1.1", PrefixLen: 24,
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	assert.NotContains(t, string(data), "gw_lrp_range", "unset range must not render")
+}
+
+func TestSpinifexTomlTemplate_PoolModeOmitsBridgeMode(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:      "node1",
+		Az:        "ap-southeast-2a",
+		Port:      "4432",
+		Region:    "ap-southeast-2",
+		BindIP:    "10.11.12.1",
+		AccessKey: "AKIATEST",
+		SecretKey: "SECRET",
+		AccountID: "123456789012",
+		NatsToken: "token",
+		ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode:  "pool",
+		ExternalIface: "enp0s3",
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "static",
+			Start: "192.168.1.150", End: "192.168.1.250",
+			Gateway: "192.168.1.1", PrefixLen: 24,
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	assert.NotContains(t, string(data), "bridge_mode", "bridged modes keep runtime auto-detect")
+}
+
+func TestIsNonBridgeableUplink(t *testing.T) {
+	for _, name := range []string{"wlan0", "wlp3s0", "wlx001122334455", "wwan0", "wwp0s20f0u6", "ppp0"} {
+		assert.True(t, isNonBridgeableUplink(name), name)
+	}
+	for _, name := range []string{"eth0", "enp0s3", "eno1", "br-wan", "veth-wan-br"} {
+		assert.False(t, isNonBridgeableUplink(name), name)
+	}
+}
+
+func TestBridgeModeFor(t *testing.T) {
+	assert.Equal(t, "nat", bridgeModeFor("nat"))
+	assert.Equal(t, "", bridgeModeFor("pool"))
+	assert.Equal(t, "", bridgeModeFor(""))
+}
+
+func TestSpinifexTomlTemplate_NATModeWithPublicPool(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:      "node1",
+		Az:        "ap-southeast-2a",
+		Port:      "4432",
+		Region:    "ap-southeast-2",
+		BindIP:    "10.11.12.1",
+		AccessKey: "AKIATEST",
+		SecretKey: "SECRET",
+		AccountID: "123456789012",
+		NatsToken: "token",
+		ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "nat",
+		BridgeMode:   "nat",
+		Pools: []admin.PoolData{
+			{Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24},
+			{
+				Name: "wan", Source: "dhcp", BindBridge: "wlan0",
+				DHCPMAC: "interface", PrefixLen: 24,
+			},
+		},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `name        = "nat-transit"`)
+	assert.Contains(t, content, `name        = "wan"`)
+	assert.Contains(t, content, `source      = "dhcp"`)
+	assert.Contains(t, content, `bind_bridge = "wlan0"`)
+	assert.Contains(t, content, `dhcp_mac    = "interface"`)
+	transitIdx := strings.Index(content, `name        = "nat-transit"`)
+	wanIdx := strings.Index(content, `name        = "wan"`)
+	assert.Less(t, transitIdx, wanIdx, "transit pool must render first (IGW allocator picks it)")
+}
+
+func TestResolvePublicPoolFlags(t *testing.T) {
+	tests := []struct {
+		name              string
+		source, poolRange string
+		bindBridge, gw    string
+		defaultBridge     string
+		wantSource        string
+		wantStart         string
+		wantEnd           string
+		wantBridge        string
+		wantErr           string
+	}{
+		{
+			name: "static pool range", source: "", poolRange: "192.168.1.150-192.168.1.250", gw: "192.168.1.1",
+			wantSource: "static", wantStart: "192.168.1.150", wantEnd: "192.168.1.250",
+		},
+		{
+			name: "dhcp defaults bind bridge", source: "dhcp", defaultBridge: "wlan0",
+			wantSource: "dhcp", wantBridge: "wlan0",
+		},
+		{
+			name: "dhcp explicit bind bridge", source: "dhcp", bindBridge: "br-wan", defaultBridge: "wlan0",
+			wantSource: "dhcp", wantBridge: "br-wan",
+		},
+		{
+			name: "dhcp rejects pool range", source: "dhcp", poolRange: "192.168.1.150-192.168.1.250",
+			wantErr: "--external-pool not allowed",
+		},
+		{
+			name: "static requires gateway", poolRange: "192.168.1.150-192.168.1.250",
+			wantErr: "--external-gateway is required",
+		},
+		{
+			name: "static rejects bind bridge", poolRange: "192.168.1.150-192.168.1.250", gw: "192.168.1.1", bindBridge: "br-wan",
+			wantErr: "--external-bind-bridge only valid",
+		},
+		{
+			name: "static requires range", source: "static", gw: "192.168.1.1",
+			wantErr: "--external-pool is required",
+		},
+		{
+			name: "bad range", poolRange: "not-a-range", gw: "192.168.1.1",
+			wantErr: "start-end IPs",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			src, start, end, bb, err := resolvePublicPoolFlags(tt.source, tt.poolRange, tt.bindBridge, tt.gw, tt.defaultBridge)
+			if tt.wantErr != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, tt.wantSource, src)
+			assert.Equal(t, tt.wantStart, start)
+			assert.Equal(t, tt.wantEnd, end)
+			assert.Equal(t, tt.wantBridge, bb)
+		})
+	}
+}
@@ -11,6 +11,7 @@ import (

 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/formation"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	toml "github.com/pelletier/go-toml/v2"
 	"github.com/spf13/cobra"
@@ -255,23 +256,6 @@ func TestWritePredastoreEncryptionKey(t *testing.T) {
 	assert.NotEqual(t, key1, key2, "per-node keys must differ")
 }

-// The viperblock key gates at-rest encryption for every volume on a fresh
-// install. masterkey.LoadShared rejects anything that isn't exactly 32 bytes at
-// 0640-or-stricter, so a regression in size/mode would silently fall back to
-// cleartext (empty path) or fail service startup.
-func TestWriteViperblockEncryptionKey(t *testing.T) {
-	configDir := t.TempDir()
-
-	keyPath, err := writeViperblockEncryptionKey(configDir)
-	require.NoError(t, err)
-	assert.Equal(t, filepath.Join(configDir, "viperblock", "encryption.key"), keyPath)
-
-	info, err := os.Stat(keyPath)
-	require.NoError(t, err)
-	assert.Equal(t, int64(32), info.Size(), "viperblock master key must be exactly 32 bytes")
-	assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "viperblock master key must be mode 0600")
-}
-
 // Joiners persist the leader's shared key via saveViperblockEncryptionKey
 // against a configDir with no viperblock/ subdir yet. The bytes round-trip
 // verbatim — the whole cluster shares one key, so any mutation would orphan
@@ -292,6 +276,181 @@ func TestSaveViperblockEncryptionKey_RoundTrip(t *testing.T) {
 	assert.Equal(t, key, got, "shared key must be written verbatim")
 }

+// A re-init must not rotate the per-node predastore key: rotating it would orphan
+// every fragment already sealed under the old key. The helper preserves an
+// existing key byte-for-byte.
+func TestWritePredastoreEncryptionKey_PreservesExisting(t *testing.T) {
+	configDir := t.TempDir()
+
+	keyPath, err := writePredastoreEncryptionKey(configDir)
+	require.NoError(t, err)
+	orig, err := os.ReadFile(keyPath)
+	require.NoError(t, err)
+
+	keyPath2, err := writePredastoreEncryptionKey(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, keyPath, keyPath2)
+	after, err := os.ReadFile(keyPath2)
+	require.NoError(t, err)
+	assert.Equal(t, orig, after, "existing predastore key must be preserved on re-init")
+}
+
+// ensureViperblockEncryptionKey generates a shared 32-byte 0600 key on a fresh
+// dir and returns bytes that match the on-disk file; a second call preserves it
+// so a re-init keeps every encrypted volume readable.
+func TestEnsureViperblockEncryptionKey(t *testing.T) {
+	configDir := t.TempDir()
+
+	key, keyPath, err := ensureViperblockEncryptionKey(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, filepath.Join(configDir, "viperblock", "encryption.key"), keyPath)
+	assert.Len(t, key, 32, "viperblock key must be 32 bytes")
+
+	info, err := os.Stat(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, int64(32), info.Size(), "viperblock master key must be exactly 32 bytes")
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "viperblock master key must be mode 0600")
+
+	onDisk, err := os.ReadFile(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, key, onDisk, "returned bytes must match the file")
+
+	key2, _, err := ensureViperblockEncryptionKey(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, key, key2, "existing viperblock key must be preserved on re-init")
+}
+
+// A present-but-corrupt viperblock key must fail loud rather than regenerate,
+// which would orphan every already-encrypted volume in the cluster.
+func TestEnsureViperblockEncryptionKey_CorruptFailsLoud(t *testing.T) {
+	configDir := t.TempDir()
+	keyDir := filepath.Join(configDir, "viperblock")
+	require.NoError(t, os.MkdirAll(keyDir, 0750))
+	keyPath := filepath.Join(keyDir, "encryption.key")
+	require.NoError(t, os.WriteFile(keyPath, []byte("bad"), 0600))
+
+	_, _, err := ensureViperblockEncryptionKey(configDir)
+	require.Error(t, err, "a truncated viperblock key must error, not regenerate")
+
+	after, err := os.ReadFile(keyPath)
+	require.NoError(t, err)
+	assert.Equal(t, []byte("bad"), after, "corrupt viperblock key must not be overwritten")
+}
+
+// ensureMasterKey generates a fresh in-memory key without touching disk (the
+// bootstrap writer persists it), and preserves an existing on-disk key so a
+// re-init never rotates the key that encrypts IAM secrets in NATS KV.
+func TestEnsureMasterKey(t *testing.T) {
+	t.Run("FreshGeneratesWithoutWriting", func(t *testing.T) {
+		configDir := t.TempDir()
+		key, existed, err := ensureMasterKey(configDir)
+		require.NoError(t, err)
+		assert.False(t, existed, "no master.key present yet")
+		assert.Len(t, key, 32)
+		assert.NoFileExists(t, filepath.Join(configDir, "master.key"),
+			"ensureMasterKey must not persist on the fresh path")
+	})
+
+	t.Run("PreservesExisting", func(t *testing.T) {
+		configDir := t.TempDir()
+		seed := make([]byte, 32)
+		for i := range seed {
+			seed[i] = byte(200 - i)
+		}
+		require.NoError(t, handlers_iam.SaveMasterKey(filepath.Join(configDir, "master.key"), seed))
+
+		key, existed, err := ensureMasterKey(configDir)
+		require.NoError(t, err)
+		assert.True(t, existed, "existing master.key must be detected")
+		assert.Equal(t, seed, key, "existing master key must be loaded verbatim")
+	})
+
+	// A present-but-corrupt master.key must fail loud, never be silently
+	// regenerated: rotating it would orphan every IAM secret in NATS KV.
+	t.Run("CorruptFailsLoud", func(t *testing.T) {
+		configDir := t.TempDir()
+		keyPath := filepath.Join(configDir, "master.key")
+		require.NoError(t, os.WriteFile(keyPath, []byte("too-short"), 0600))
+
+		_, _, err := ensureMasterKey(configDir)
+		require.Error(t, err, "a truncated master.key must error, not regenerate")
+
+		after, err := os.ReadFile(keyPath)
+		require.NoError(t, err)
+		assert.Equal(t, []byte("too-short"), after, "corrupt master key must not be overwritten")
+	})
+}
+
+// The identity bundle is preserved holistically on re-init: the load helpers
+// recover the exact system credentials and never rewrite bootstrap.json. Admin
+// credentials are not recovered from disk — awsgw consumes and deletes
+// bootstrap.json after first boot, so the preserve path must not read it back.
+func TestPreservedIdentityBundle(t *testing.T) {
+	configDir := t.TempDir()
+	bootstrapDir := filepath.Join(configDir, "awsgw")
+
+	masterKey, err := handlers_iam.GenerateMasterKey()
+	require.NoError(t, err)
+	const (
+		sysAccess   = "AKIASYSTEM0000000000"
+		sysSecret   = "system-secret-value"
+		adminAccess = "AKIAADMIN00000000000"
+		adminSecret = "admin-secret-value"
+		accountID   = "123456789012"
+	)
+	require.NoError(t, writeBootstrapFilesWithAdmin(configDir, bootstrapDir, masterKey,
+		sysAccess, sysSecret, accountID, adminAccess, adminSecret))
+	require.NoError(t, writeSystemCredentials(configDir, sysAccess, sysSecret))
+
+	bootstrapPath := filepath.Join(bootstrapDir, "bootstrap.json")
+	bootstrapBefore, err := os.ReadFile(bootstrapPath)
+	require.NoError(t, err)
+
+	// master.key round-trips and is flagged as pre-existing.
+	loadedKey, existed, err := ensureMasterKey(configDir)
+	require.NoError(t, err)
+	assert.True(t, existed)
+	assert.Equal(t, masterKey, loadedKey)
+
+	// System credentials come back verbatim (they must match the NATS KV seed).
+	gotSysAccess, gotSysSecret, err := loadSystemCredentials(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, sysAccess, gotSysAccess)
+	assert.Equal(t, sysSecret, gotSysSecret)
+
+	// The preserve helpers must not rewrite the seed file.
+	bootstrapAfter, err := os.ReadFile(bootstrapPath)
+	require.NoError(t, err)
+	assert.Equal(t, bootstrapBefore, bootstrapAfter, "bootstrap.json must not be rewritten on re-init")
+}
+
+// loadSystemCredentials surfaces a clear error when the file is absent rather
+// than returning empty credentials that would silently break SigV4 auth.
+func TestLoadSystemCredentials_Missing(t *testing.T) {
+	_, _, err := loadSystemCredentials(t.TempDir())
+	require.Error(t, err)
+}
+
+// A present-but-empty or malformed system-credentials.json must error rather
+// than yield blank credentials that would silently break SigV4 between services.
+func TestLoadSystemCredentials_Invalid(t *testing.T) {
+	t.Run("EmptyValues", func(t *testing.T) {
+		configDir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(configDir, "system-credentials.json"),
+			[]byte(`{"access_key_id":"","secret_access_key":""}`), 0600))
+		_, _, err := loadSystemCredentials(configDir)
+		require.Error(t, err)
+	})
+
+	t.Run("MalformedJSON", func(t *testing.T) {
+		configDir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(configDir, "system-credentials.json"),
+			[]byte("not json"), 0600))
+		_, _, err := loadSystemCredentials(configDir)
+		require.Error(t, err)
+	})
+}
+
 // A fresh install must render encryption_key_file so viperblockd enables
 // at-rest encryption; an empty field (upgrade / legacy) must omit the line
 // entirely so the volume stays in cleartext legacy mode.
@@ -382,20 +541,19 @@ func TestSpinifexTomlTemplate_ExternalPoolDHCPSource(t *testing.T) {
 	dir := t.TempDir()
 	path := filepath.Join(dir, "spinifex.toml")
 	settings := admin.ConfigSettings{
-		Node:           "node1",
-		Az:             "ap-southeast-2a",
-		Port:           "4432",
-		Region:         "ap-southeast-2",
-		BindIP:         "10.11.12.1",
-		ConfigDir:      dir,
-		OVNNBAddr:      "tcp:127.0.0.1:6641",
-		OVNSBAddr:      "tcp:127.0.0.1:6642",
-		ExternalMode:   "pool",
-		ExternalIface:  "eth0",
-		PoolName:       "wan",
-		PoolSource:     "dhcp",
-		PoolBindBridge: "br-wan",
-		PoolPrefixLen:  24,
+		Node:          "node1",
+		Az:            "ap-southeast-2a",
+		Port:          "4432",
+		Region:        "ap-southeast-2",
+		BindIP:        "10.11.12.1",
+		ConfigDir:     dir,
+		OVNNBAddr:     "tcp:127.0.0.1:6641",
+		OVNSBAddr:     "tcp:127.0.0.1:6642",
+		ExternalMode:  "pool",
+		ExternalIface: "eth0",
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "dhcp", BindBridge: "br-wan", PrefixLen: 24,
+		}},
 	}
 	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))

@@ -424,12 +582,11 @@ func TestSpinifexTomlTemplate_ExternalPoolStaticSource(t *testing.T) {
 		OVNSBAddr:     "tcp:127.0.0.1:6642",
 		ExternalMode:  "pool",
 		ExternalIface: "eth0",
-		PoolName:      "wan",
-		PoolSource:    "static",
-		PoolStart:     "192.168.1.150",
-		PoolEnd:       "192.168.1.250",
-		PoolGateway:   "192.168.1.1",
-		PoolPrefixLen: 24,
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "static",
+			Start: "192.168.1.150", End: "192.168.1.250",
+			Gateway: "192.168.1.1", PrefixLen: 24,
+		}},
 	}
 	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))

@@ -454,8 +611,9 @@ func TestApplyNetworkConfig_PropagatesPoolBindBridge(t *testing.T) {
 		PoolPrefixLen:  24,
 	}
 	applyNetworkConfig(settings, nc)
-	assert.Equal(t, "dhcp", settings.PoolSource)
-	assert.Equal(t, "br-wan", settings.PoolBindBridge)
+	require.Len(t, settings.Pools, 1)
+	assert.Equal(t, "dhcp", settings.Pools[0].Source)
+	assert.Equal(t, "br-wan", settings.Pools[0].BindBridge)
 }

 func renderSingleNodePredastore(t *testing.T, settings admin.ConfigSettings) string {
@@ -123,7 +123,7 @@ func runClusterShutdown(cmd *cobra.Command, args []string) {
 		phaseStart := time.Now()
 		fmt.Printf("[%s] Sending to %d node(s)...\n", strings.ToUpper(phase), nodeCount)

-		acks, err := collectShutdownACKs(nc, topic, reqData, nodeCount, timeout)
+		acks, err := collectShutdownACKs(nc, topic, reqData, nodeCount, "", timeout)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "[%s] Error: %v\n", strings.ToUpper(phase), err)
 			if !force {
@@ -216,36 +216,14 @@ func runNodeDrainLocal(cmd *cobra.Command, args []string) {
 // from the local node, ignoring ACKs from any other node. Returns an error if
 // the local node does not respond within timeout.
 func collectLocalShutdownACK(nc *nats.Conn, topic string, reqData []byte, node string, timeout time.Duration) (daemon.ShutdownACK, error) {
-	inbox := nats.NewInbox()
-	sub, err := nc.SubscribeSync(inbox)
+	acks, err := collectShutdownACKs(nc, topic, reqData, 1, node, timeout)
 	if err != nil {
-		return daemon.ShutdownACK{}, fmt.Errorf("failed to subscribe to inbox: %w", err)
+		return daemon.ShutdownACK{}, err
 	}
-	defer sub.Unsubscribe()
-
-	if err := nc.PublishRequest(topic, inbox, reqData); err != nil {
-		return daemon.ShutdownACK{}, fmt.Errorf("failed to publish request: %w", err)
-	}
-	nc.Flush()
-
-	deadline := time.Now().Add(timeout)
-	for {
-		remaining := time.Until(deadline)
-		if remaining <= 0 {
-			return daemon.ShutdownACK{}, fmt.Errorf("timeout waiting for local node %q ACK", node)
-		}
-		msg, err := sub.NextMsg(remaining)
-		if err != nil {
-			return daemon.ShutdownACK{}, fmt.Errorf("timeout waiting for local node %q ACK", node)
-		}
-		var ack daemon.ShutdownACK
-		if err := json.Unmarshal(msg.Data, &ack); err != nil {
-			continue
-		}
-		if ack.Node == node {
-			return ack, nil
-		}
+	if len(acks) == 0 {
+		return daemon.ShutdownACK{}, fmt.Errorf("timeout waiting for local node %q ACK", node)
 	}
+	return acks[0], nil
 }

 // runClusterDrainDHCP asks every vpcd to DHCPRELEASE all external-pool leases
@@ -322,7 +300,9 @@ func drainDHCPLeases(nc *nats.Conn, timeout time.Duration) (released, responders
 }

 // collectShutdownACKs publishes a shutdown request and collects ACKs from nodes.
-func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount int, timeout time.Duration) ([]daemon.ShutdownACK, error) {
+// When nodeFilter is non-empty, only ACKs from that node count toward nodeCount
+// and all others are ignored (used by the single-node local-drain path).
+func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount int, nodeFilter string, timeout time.Duration) ([]daemon.ShutdownACK, error) {
 	inbox := nats.NewInbox()
 	sub, err := nc.SubscribeSync(inbox)
 	if err != nil {
@@ -350,6 +330,9 @@ func collectShutdownACKs(nc *nats.Conn, topic string, reqData []byte, nodeCount
 		if err := json.Unmarshal(msg.Data, &ack); err != nil {
 			continue
 		}
+		if nodeFilter != "" && ack.Node != nodeFilter {
+			continue
+		}
 		acks = append(acks, ack)
 	}
 	return acks, nil
@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd

 import (
@@ -154,6 +138,12 @@ func formatRoles(resp types.NodeStatusResponse) string {
 	if resp.NATSRole != "" {
 		roles = append(roles, "nats:"+resp.NATSRole)
 	}
+	if resp.OVNNBRole != "" {
+		roles = append(roles, "ovn-nb:"+resp.OVNNBRole)
+	}
+	if resp.OVNSBRole != "" {
+		roles = append(roles, "ovn-sb:"+resp.OVNSBRole)
+	}
 	if resp.PredastoreRole != "" {
 		roles = append(roles, "predastore:"+resp.PredastoreRole)
 	}
@@ -38,6 +38,26 @@ func TestFormatRoles(t *testing.T) {
 			resp: types.NodeStatusResponse{NATSRole: "leader", PredastoreRole: "leader"},
 			want: "nats:leader,predastore:leader",
 		},
+		{
+			name: "ovn roles only",
+			resp: types.NodeStatusResponse{OVNNBRole: "leader", OVNSBRole: "follower"},
+			want: "ovn-nb:leader,ovn-sb:follower",
+		},
+		{
+			name: "all four roles ordered",
+			resp: types.NodeStatusResponse{
+				NATSRole:       "leader",
+				OVNNBRole:      "leader",
+				OVNSBRole:      "follower",
+				PredastoreRole: "follower",
+			},
+			want: "nats:leader,ovn-nb:leader,ovn-sb:follower,predastore:follower",
+		},
+		{
+			name: "ovn-nb leader with nats, sb absent",
+			resp: types.NodeStatusResponse{NATSRole: "follower", OVNNBRole: "leader"},
+			want: "nats:follower,ovn-nb:leader",
+		},
 	}

 	for _, tt := range tests {
@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd

 import (
@@ -1,32 +1,20 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd

 import (
+	"context"
 	"fmt"
 	"log/slog"
 	"os"
 	"strings"

 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/network/external"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"github.com/mulgadc/spinifex/spinifex/service"
 	"github.com/mulgadc/spinifex/spinifex/services/awsgw"
 	"github.com/mulgadc/spinifex/spinifex/services/nats"
 	"github.com/mulgadc/spinifex/spinifex/services/predastore"
+	"github.com/mulgadc/spinifex/spinifex/services/qmpcollector"
 	"github.com/mulgadc/spinifex/spinifex/services/spinifexui"
 	"github.com/mulgadc/spinifex/spinifex/services/viperblockd"
 	"github.com/mulgadc/spinifex/spinifex/vpcd"
@@ -39,6 +27,28 @@ var serviceCmd = &cobra.Command{
 	Short: "Manage Spinifex services",
 }

+// initTelemetry installs the JSON slog default (trace-stamping) and the OTel
+// providers for a service process. The returned func flushes exporters and
+// must be deferred; with no OTLP endpoint configured both are no-ops.
+func initTelemetry(serviceName string, debug bool) func() {
+	level := slog.LevelInfo
+	if debug {
+		level = slog.LevelDebug
+	}
+	otelsetup.SetDefaultJSONLogger(level)
+
+	shutdown, err := otelsetup.Init(context.Background(), serviceName)
+	if err != nil {
+		slog.Warn("otel init", "service", serviceName, "error", err)
+		return func() {}
+	}
+	return func() {
+		if err := shutdown(context.Background()); err != nil {
+			slog.Warn("otel shutdown", "service", serviceName, "error", err)
+		}
+	}
+}
+
 var predastoreCmd = &cobra.Command{
 	Use:   "predastore",
 	Short: "Manage the predastore service",
@@ -126,6 +136,8 @@ var predastoreStartCmd = &cobra.Command{
 		pprofEnabled := viper.GetBool("pprof")
 		pprofOutput := viper.GetString("pprof-output")

+		defer initTelemetry("predastore", debug)()
+
 		service, err := service.New("predastore", &predastore.Config{
 			Port:       port,
 			Host:       host,
@@ -288,6 +300,8 @@ var viperblockStartCmd = &cobra.Command{
 			encryptionKeyFile = envKey
 		}

+		defer initTelemetry("viperblockd", false)()
+
 		service, err := service.New("viperblock", &viperblockd.Config{
 			NatsHost:          nodeConfig.NATS.Host,
 			NatsToken:         nodeConfig.NATS.ACL.Token,
@@ -366,6 +380,8 @@ var natsStartCmd = &cobra.Command{

 		cfgFile := viper.GetString("config")

+		defer initTelemetry("nats", debug)()
+
 		service, err := service.New("nats", &nats.Config{
 			ConfigFile: cfgFile,
 			Port:       port,
@@ -459,6 +475,8 @@ var spinifexStartCmd = &cobra.Command{
 		// Apply changes back to cluster config
 		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig

+		defer initTelemetry("spinifex-daemon", false)()
+
 		svc, err := service.New("spinifex", clusterConfig)

 		if err != nil {
@@ -564,6 +582,8 @@ var awsgwStartCmd = &cobra.Command{
 		// Apply changes back to cluster config
 		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig

+		defer initTelemetry("awsgw", viper.GetBool("debug"))()
+
 		awsgw.SetBuildInfo(Version, Commit)
 		service, err := service.New("awsgw", clusterConfig)

@@ -621,6 +641,8 @@ var spinifexUIStartCmd = &cobra.Command{
 		tlsCert := viper.GetString("spinifex-ui-tls-cert")
 		tlsKey := viper.GetString("spinifex-ui-tls-key")

+		defer initTelemetry("spinifex-ui", false)()
+
 		svc, err := service.New("spinifex-ui", &spinifexui.Config{
 			Port:    port,
 			Host:    host,
@@ -730,12 +752,13 @@ var vpcdStartCmd = &cobra.Command{
 		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]

 		// Map cluster-wide external pools to vpcd config
-		var extPools []vpcd.ExternalPoolConfig
+		var extPools []external.ExternalPoolConfig
 		for _, p := range clusterConfig.Network.ExternalPools {
-			extPools = append(extPools, vpcd.ExternalPoolConfig{
+			extPools = append(extPools, external.ExternalPoolConfig{
 				Name:            p.Name,
 				Source:          p.Source,
 				BindBridge:      p.BindBridge,
+				DHCPMAC:         p.DHCPMAC,
 				RangeStart:      p.RangeStart,
 				RangeEnd:        p.RangeEnd,
 				Gateway:         p.Gateway,
@@ -767,6 +790,8 @@ var vpcdStartCmd = &cobra.Command{
 			nodeConfig.BaseDir = baseDir
 		}

+		defer initTelemetry("vpcd", false)()
+
 		svc, err := service.New("vpcd", &vpcd.Config{
 			NatsHost:          nodeConfig.NATS.Host,
 			NatsToken:         nodeConfig.NATS.ACL.Token,
@@ -781,6 +806,7 @@ var vpcdStartCmd = &cobra.Command{
 			ExternalInterface: nodeConfig.VPCD.ExternalInterface,
 			BridgeMode:        nodeConfig.VPCD.BridgeMode,
 			AZ:                nodeConfig.AZ,
+			NATExemptCIDRs:    clusterConfig.Network.NATExemptCIDRs,
 		})
 		if err != nil {
 			fmt.Println("Error starting vpcd service:", err)
@@ -823,6 +849,77 @@ var vpcdStatusCmd = &cobra.Command{
 	},
 }

+var qmpCollectorCmd = &cobra.Command{
+	Use:   "qmp-collector",
+	Short: "Manage the qmp-collector (guest metrics) service",
+}
+
+var qmpCollectorStartCmd = &cobra.Command{
+	Use:   "start",
+	Short: "Start the qmp-collector service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("Starting qmp-collector service...")
+
+		cfgFile := viper.GetString("config")
+		if cfgFile == "" {
+			fmt.Println("Config file is not set")
+			return
+		}
+		clusterConfig, err := config.LoadConfig(cfgFile)
+		if err != nil {
+			fmt.Println("Error loading config file:", err)
+			return
+		}
+		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
+
+		defer initTelemetry("qmp-collector", false)()
+
+		svc, err := service.New("qmp-collector", &qmpcollector.Config{
+			NatsHost:   nodeConfig.NATS.Host,
+			NatsToken:  nodeConfig.NATS.ACL.Token,
+			NatsCACert: nodeConfig.NATS.CACert,
+			BaseDir:    nodeConfig.BaseDir,
+			NodeName:   clusterConfig.Node,
+		})
+		if err != nil {
+			fmt.Println("Error starting qmp-collector service:", err)
+			return
+		}
+		if _, err = svc.Start(); err != nil {
+			fmt.Println("Error starting qmp-collector service:", err)
+			os.Exit(1)
+		}
+		fmt.Println("qmp-collector service started")
+	},
+}
+
+var qmpCollectorStopCmd = &cobra.Command{
+	Use:   "stop",
+	Short: "Stop the qmp-collector service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("Stopping qmp-collector service...")
+
+		svc, err := service.New("qmp-collector", &qmpcollector.Config{})
+		if err != nil {
+			fmt.Println("Error stopping qmp-collector service:", err)
+			return
+		}
+		if err = svc.Stop(); err != nil {
+			fmt.Println("Error stopping qmp-collector service:", err)
+			os.Exit(1)
+		}
+		fmt.Println("qmp-collector service stopped")
+	},
+}
+
+var qmpCollectorStatusCmd = &cobra.Command{
+	Use:   "status",
+	Short: "Get status of the qmp-collector service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("qmp-collector service status: ...")
+	},
+}
+
 func init() {
 	viper.SetEnvPrefix("SPINIFEX") // Prefix for environment variables
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
@@ -873,11 +970,6 @@ func init() {
 	viper.BindEnv("encryption-key-file", "SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE")
 	viper.BindPFlag("encryption-key-file", predastoreCmd.PersistentFlags().Lookup("encryption-key-file"))

-	// Predastore Backend
-	predastoreCmd.PersistentFlags().String("backend", "distributed", "Predastore (S3) backend")
-	viper.BindEnv("backend", "SPINIFEX_PREDASTORE_BACKEND")
-	viper.BindPFlag("backend", predastoreCmd.PersistentFlags().Lookup("backend"))
-
 	// Predastore Node ID. Default -1 is dev mode (launch every configured
 	// QUIC node in-process). Production deployments set this to the node's
 	// real ID (>= 1) via SPINIFEX_PREDASTORE_NODE_ID or --node-id.
@@ -1022,4 +1114,11 @@ func init() {
 	vpcdCmd.AddCommand(vpcdStartCmd)
 	vpcdCmd.AddCommand(vpcdStopCmd)
 	vpcdCmd.AddCommand(vpcdStatusCmd)
+
+	// qmp-collector
+	serviceCmd.AddCommand(qmpCollectorCmd)
+
+	qmpCollectorCmd.AddCommand(qmpCollectorStartCmd)
+	qmpCollectorCmd.AddCommand(qmpCollectorStopCmd)
+	qmpCollectorCmd.AddCommand(qmpCollectorStatusCmd)
 }
@@ -0,0 +1,24 @@
+package cmd
+
+import "testing"
+
+func TestInitTelemetryNoEndpoint(t *testing.T) {
+	t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
+
+	tests := []struct {
+		name  string
+		debug bool
+	}{
+		{"info level", false},
+		{"debug level", true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			flush := initTelemetry("test-svc", tt.debug)
+			if flush == nil {
+				t.Fatal("initTelemetry returned nil flush func")
+			}
+			flush()
+		})
+	}
+}
@@ -16,28 +16,37 @@ internal_suffix = "spinifex.internal"
 ipsec_enabled = {{.IPSecEnabled}}
 {{- if .ExternalMode}}
 external_mode = "{{.ExternalMode}}"
+{{- range .Pools}}

 [[network.external_pools]]
-name        = "{{.PoolName}}"
-{{- if .PoolSource}}
-source      = "{{.PoolSource}}"
+name        = "{{.Name}}"
+{{- if .Source}}
+source      = "{{.Source}}"
 {{- end}}
-{{- if .PoolBindBridge}}
-bind_bridge = "{{.PoolBindBridge}}"
+{{- if .BindBridge}}
+bind_bridge = "{{.BindBridge}}"
 {{- end}}
-{{- if .PoolStart}}
-range_start = "{{.PoolStart}}"
-range_end   = "{{.PoolEnd}}"
+{{- if .DHCPMAC}}
+dhcp_mac    = "{{.DHCPMAC}}"
 {{- end}}
-{{- if .PoolGateway}}
-gateway     = "{{.PoolGateway}}"
+{{- if .Start}}
+range_start = "{{.Start}}"
+range_end   = "{{.End}}"
 {{- end}}
-{{- if .PoolGatewayIP}}
-gateway_ip  = "{{.PoolGatewayIP}}"
+{{- if .Gateway}}
+gateway     = "{{.Gateway}}"
+{{- end}}
+{{- if .GatewayIP}}
+gateway_ip  = "{{.GatewayIP}}"
+{{- end}}
+{{- if .GwLrpRangeStart}}
+gw_lrp_range_start = "{{.GwLrpRangeStart}}"
+gw_lrp_range_end   = "{{.GwLrpRangeEnd}}"
+{{- end}}
+prefix_len  = {{if .PrefixLen}}{{.PrefixLen}}{{else}}24{{end}}
+{{- if .DNSServers}}
+dns_servers = [{{range $i, $s := .DNSServers}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
 {{- end}}
-prefix_len  = {{if .PoolPrefixLen}}{{.PoolPrefixLen}}{{else}}24{{end}}
-{{- if .PoolDNSServers}}
-dns_servers = [{{range $i, $s := .PoolDNSServers}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
 {{- end}}
 {{- end}}
 {{- if .BootstrapVpcId}}
@@ -107,6 +116,9 @@ ovn_sb_addr = "{{.OVNSBAddr}}"
 {{- if .ExternalIface}}
 external_interface = "{{.ExternalIface}}"
 {{- end}}
+{{- if .BridgeMode}}
+bridge_mode = "{{.BridgeMode}}"
+{{- end}}

 [nodes.{{.Node}}.predastore]
 host = "{{.BindIP}}:8443"
@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd

 import (
@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package main

 import (
@@ -31,6 +31,9 @@ Service lifecycle commands for starting, stopping, and checking status of all Sp
 | `spx service vpcd start` | — | Loads cluster config → starts VPC daemon (subscribes to `vpc.*` NATS events, translates to OVN logical switches/ports/routers) |
 | `spx service vpcd stop` | — | Stops the vpcd service |
 | `spx service vpcd status` | — | Reports vpcd service status |
+| `spx service qmp-collector start` | — | Starts the guest-metrics collector (polls per-VM telemetry QMP sockets + tap counters, publishes CloudWatch-shaped series to NATS `metrics.ec2.*`) |
+| `spx service qmp-collector stop` | — | Stops the qmp-collector service |
+| `spx service qmp-collector status` | — | Reports qmp-collector service status |

 ### Cluster Inspection

@@ -51,7 +54,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t

 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force` | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` |
+| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; add `--external-pool start-end --external-gateway <ip>` or `--external-source=dhcp [--external-bind-bridge <iface>]` for a public pool with full EIP support; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 — they require `pool` mode), `--no-external` | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/`. `--force` on an already-initialized node is idempotent for crypto: it preserves the existing keys (master.key, predastore/viperblock encryption keys), system + admin credentials, and CA, and refreshes only the config files and the CA-signed server certificate (so a changed bind IP / SANs is picked up without breaking already-joined nodes or CA-baked AMIs). A genuine clean slate requires removing the data dirs first. |
 | `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations |

 ### Version
@@ -93,8 +96,69 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 |---------|-------|-------------|
 | `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify` | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. On every import a best-effort `virt-customize` step bakes the deployment CA (`<data-root>/config/ca.pem`) into the image's trust store before the block copy, so in-guest SDK-over-TLS calls to Spinifex endpoints trust the gateway from first boot; an image libguestfs cannot customize is imported as-is (CA-free) and logs a skip. Because the CA is fixed into the image, rotating the cluster CA requires re-importing affected images. |
 | `spx admin images list` | — | Lists available OS images that can be imported or downloaded |
+| `spx admin images promote` | `--image-id` (required), `--yes` | Reads `ami-<id>/config.json`, validates the AMI is account-owned, then rewrites `ImageOwnerAlias` to `"system"` in-place. No block data is copied. The change takes effect immediately — the AMI becomes visible to all accounts via `DescribeImages`. Prompts for confirmation (skipped with `--yes`). Already-system AMIs are refused. |
 | `spx admin images remove` | `--image-id` (required), `--force`, `--yes` | Loads `ami-<id>/config.json`, walks transitive dependents — copied snapshots whose `VolumeID == imageID`, volumes whose `SnapshotID` references the internal `snap-ami-<id>` or any derived snap, and account AMIs created via `CopyImage` whose `SnapshotID` is a derived snap — then prompts (skipped with `--yes`) before deleting `ami-<id>/config.json` (the DescribeImages barrier) followed by the rest of `ami-<id>/` and `snap-ami-<id>/`. Account-owned AMIs are refused with a hint pointing at `aws ec2 deregister-image` + `aws ec2 delete-snapshot`. `--force` bypasses the dependency, ownership and config-corrupt checks for salvage of orphaned blocks. |

+#### Image integrity verification (CMMC SI.L1-3.14.2)
+
+Catalog imports (`spx admin images import --name <name>`) verify the image
+against the catalog-declared SHA-256/SHA-512 digest before extraction. The sums
+file is fetched from the catalog `Checksum` URL over HTTPS only (cross-scheme
+redirects refused), and verification runs on both fresh downloads and cache
+hits so a poisoned cache is caught on the next import.
+
+On mismatch the import exits non-zero, the cached file is left on disk for
+inspection, and the printed guidance is `spx admin images import --name <name>
+--force` to re-download.
+
+`--file` imports skip verification by design: operator-supplied media is
+outside Spinifex's trust boundary and the operator is responsible for
+integrity (e.g. `sha256sum` against a trusted upstream digest before import).
+The skip is recorded as an INFO `slog` event with `reason=local-file-import`
+so a CMMC assessor can audit the decision from journald.
+
+`--skip-verify` bypasses the checksum step for catalog imports. The command
+still downloads via the catalog URL but does not compare the image digest
+against the sums file. Intended for narrow cases such as debugging upstream
+mirror issues or running against a transiently-broken `latest/` path; the
+skip is logged at WARN with `reason=skip-verify-flag` and printed to stderr
+so operators and assessors see it. Prefer `--file` with an out-of-band
+verified image over `--skip-verify` whenever possible.
+
+**Limitation:** verification confirms the image matches the digest the mirror
+served. A mirror compromise that swaps both image and sums file is not
+detected; closing that gap requires GPG signature verification of the sums
+file, deferred to a later phase.
+
+#### `spx admin images remove` caveats
+
+Admin-imported AMIs (`ImageOwnerAlias = "system"`) live
+under the `ami-<id>/` S3 prefix and use a viperblock-internal snap checkpoint
+at `snap-ami-<id>/` — there is no `snap-<id>/metadata.json`. The AWS handlers
+(`DeregisterImage`, `DeleteSnapshot`) reject system owners with
+`UnauthorizedOperation`, which is the right behaviour for tenant API callers
+but leaves no AWS-flow path to reclaim space. `spx admin images remove` is
+the admin-trust-boundary counterpart that performs the dependency walk and
+hard-deletes the blocks directly against predastore.
+
+`CopyImage` of a system AMI is metadata-only: it writes a fresh
+`snap-<acct>/metadata.json` whose `VolumeID` points at `ami-<sys>` and a new
+`ami-<acct>/config.json` referencing that snap. Volumes launched from the
+copied AMI read transitively from `ami-<sys>/chunks/...`. The remove command
+walks this transitive set and refuses if anything references the target.
+
+**TOCTOU window:** between the safety scan and the `config.json` delete a
+concurrent `RunInstances` against the AMI could create a new dependent
+volume. The window is sub-second on a healthy cluster. The admin running
+this command is expected to know the fleet's operational state; if the race
+fires the result is a `vol-<id>` with deleted backing blocks, recovered by
+terminating the orphaned instance.
+
+**`--force` bypasses every safety check** (dependents, ownership, missing /
+corrupt `config.json`). Use only for salvage of orphaned blocks. Running it
+against a live system AMI corrupts every dependent volume on the next disk
+read.
+
 ### GPU Management

 | Command | Flags | Description |
@@ -107,6 +171,11 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `spx admin gpu mig enable` | `--profile <name>` (required, e.g. `1g.10gb`), `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → enables MIG mode on each target (`gpu.EnableMIGMode`) → lists available profiles (`gpu.ListProfiles`) and validates requested profile name → destroys any existing instances (`gpu.DestroyAllInstances`) → creates new instances filling GPU capacity (`gpu.CreateInstances`) → writes `mig_profile` to `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. |
 | `spx admin gpu mig disable` | `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → destroys all GPU instances (`gpu.DestroyAllInstances`) → disables MIG mode (`gpu.DisableMIGMode`) → clears `mig_profile` in `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. |

+### EKS Control-Plane Disaster Recovery
+
+| Command | Flags | Description |
+|---------|-------|-------------|
+| `spx admin eks restore-snapshot` | `--cluster` (required), `--snapshot` (optional, defaults to the latest snapshot in predastore), `--account` (optional, defaults to the bootstrap account) | Single-CP total-loss DR path (fail-safe): validates the snapshot exists in predastore BEFORE any mutation (a typo'd/missing key hard-fails, never resets into an empty datastore) → launches a fresh control-plane VM as a cluster-init seed (replaying the persisted create-time launch template) → sets a required-snapshot `RecoveryDirective` (`cluster-reset`) so the boot-time recovery agent aborts rather than resets-into-empty if it cannot fetch the snapshot → persists the replacement in cluster meta BEFORE re-pointing the NLB (so an NLB failure is convergeable by the reconciler, returned as a provisional status, not a hard error) → re-points the cluster NLB's apiserver and konnectivity target groups from the old CP's ENI to the new one → fences the old CP with retries, failing loudly if it cannot be confirmed terminated (split-brain guard). Any failure before the meta commit unwinds the fresh CP (terminate + clear directive) so a re-run does not stack a second resetting control plane. The returned status is provisional — success means the sequence completed, not that etcd is restored and serving; verify cluster health. HA clusters (a spread with a potentially surviving quorum) are rejected — recover those via quorum reformation instead. |

 ## AWS-Compatible API

@@ -114,7 +183,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `run-instances` | `--image-id`, `--instance-type`, `--count`, `--key-name`, `--user-data`, `--subnet-id`, `--security-group-ids`, `--tag-specifications` (instance-scoped), `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination), `--placement` (GroupName), `--iam-instance-profile` (Name/Arn), `--capacity-reservation-specification` (CapacityReservationTarget.CapacityReservationId, targeted-by-id only), `--metadata-options` (HttpPutResponseHopLimit; IMDSv2-only enforced — rejects `http-tokens=optional`) | `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--launch-template`, `--hibernate-options` | **DONE** |
+| `run-instances` | `--image-id`, `--instance-type`, `--count`, `--key-name`, `--user-data`, `--subnet-id`, `--security-group-ids`, `--tag-specifications` (instance-scoped), `--block-device-mappings` (DeviceName, VolumeSize, VolumeType, Iops, DeleteOnTermination), `--placement` (GroupName), `--iam-instance-profile` (Name/Arn), `--capacity-reservation-specification` (CapacityReservationTarget.CapacityReservationId, targeted-by-id only), `--metadata-options` (HttpPutResponseHopLimit; IMDSv2-only enforced — rejects `http-tokens=optional`), `--launch-template` (LaunchTemplateId/LaunchTemplateName, Version — resolves `$Default`/`$Latest`; direct params override the template) | `--dry-run`, `--client-token`, `--disable-api-termination`, `--ebs-optimized`, `--network-interfaces`, `--private-ip-address`, `--monitoring`, `--credit-specification`, `--cpu-options`, `--hibernate-options` | **DONE** |
 | `describe-instances` | `--instance-ids`, `--filters` (instance-state-name, instance-id, instance-type, vpc-id, subnet-id, tag:*, tag-key, tag-value) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
 | `start-instances` | `--instance-ids` | `--dry-run`, `--force` | **DONE** |
 | `stop-instances` | `--instance-ids` | `--force`, `--hibernate`, `--dry-run` | **DONE** |
@@ -130,29 +199,20 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `monitor-instances` | — | `--instance-ids` | **NOT STARTED** |
 | `unmonitor-instances` | — | `--instance-ids` | **NOT STARTED** |

-`run-instances --iam-instance-profile` enforces `iam:PassRole` on the contained
-role ARN and rejects cross-account profile ARNs. Placement strategies: `spread`
-(1 per node, atomic CAS) and `cluster` (pin all to one node).
+### EC2 — Spot Instances

-`run-instances --capacity-reservation-specification` consumes a reserved slot on
-the node owning the targeted reservation (targeted-by-id only; `open`
-auto-matching is not supported). The combination with `--placement` (group) is
-rejected. `describe-instances` then reports the instance's `CapacityReservationId`.
+Spot Instance Requests (SIRs) are a **mock** over the on-demand `run-instances` path: a request synchronously launches real VMs on the operator's own compute and is then reported `active`/`fulfilled`. There is no spot market — no bidding, price rejection, interruption, or reclamation, and instances are never reclaimed.

-`modify-instance-metadata-options` is the only metadata-options mutator — AWS has
-no `get-instance-metadata-options` API, so the per-instance block is read through
-`describe-instances`. Only the hop limit is mutable; `--http-tokens optional` and
-the other posture-weakening values are refused with `UnsupportedOperation`, since
-the platform enforces IMDSv2 permanently. `run-instances --metadata-options` runs
-the same validation at launch.
+| Command | Implemented Flags | Missing Flags | Status |
+|---------|-------------------|---------------|--------|
+| `request-spot-instances` | `--instance-count` (default 1), `--type` (`one-time`/`persistent` — stored, behaviour identical), `--spot-price` (echoed only), `--client-token`, `--launch-specification` (ImageId, InstanceType, KeyName, SubnetId, SecurityGroupIds, UserData, BlockDeviceMappings, IamInstanceProfile, Placement.GroupName, NetworkInterfaces), `--tag-specifications` (spot-instances-request) | `--valid-from`, `--valid-until`, `--launch-group`, `--availability-zone-group`, `--block-duration-minutes`, `--instance-interruption-behavior`, `--dry-run` | **DONE** (mock) |
+| `describe-spot-instance-requests` | `--spot-instance-request-ids`, `--filters` (spot-instance-request-id, state, instance-id, launch.image-id, launch.instance-type, launch.key-name, type, launched-availability-zone, tag-key, tag:*) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
+| `cancel-spot-instance-requests` | `--spot-instance-request-ids` | `--dry-run` | **DONE** |
+
+`describe-spot-price-history` is **unsupported** (returns `InvalidAction`): on owned hardware there is no spot/on-demand price differential, so any synthetic price would be misleading rather than helpful.

 ### EC2 — IAM Instance Profile Associations

-Stored as `vm.VM.IamInstanceProfileArn` + `IamInstanceProfileAssociationId`
-(one ARN per instance). Association IDs (`iip-assoc-`) are regenerated on every
-Associate/Replace and never reused. Auto-disassociated on terminate; preserved
-across stop/start.
-
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
 | `associate-iam-instance-profile` | `--instance-id`, `--iam-instance-profile` (Name/Arn) | `--dry-run` | **DONE** |
@@ -164,10 +224,10 @@ across stop/start.

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-key-pair` | `--key-name`, `--key-type` (rsa/ed25519) | `--key-format`, `--tag-specifications`, `--dry-run` | **DONE** |
+| `create-key-pair` | `--key-name`, `--key-type` (rsa/ed25519), `--tag-specifications` | `--key-format`, `--dry-run` | **DONE** |
 | `describe-key-pairs` | `--key-names`, `--key-pair-ids`, `--filters` (key-pair-id, key-name, fingerprint, tag:*) | `--max-results`, `--dry-run` | **DONE** |
 | `delete-key-pair` | `--key-name`, `--key-pair-id` | `--dry-run` | **DONE** |
-| `import-key-pair` | `--key-name`, `--public-key-material` | `--tag-specifications`, `--dry-run` | **DONE** |
+| `import-key-pair` | `--key-name`, `--public-key-material`, `--tag-specifications` | `--dry-run` | **DONE** |

 ### EC2 — AMI Images

@@ -183,16 +243,14 @@ across stop/start.
 | `reset-image-attribute` | `--image-id`, `--attribute description` | `--attribute launchPermission`, `--dry-run` | **DONE** |
 | `import-image` | — | `--disk-containers`, `--description`, `--architecture`, `--platform` | **NOT STARTED** |

-`copy-image` is metadata-only — no block copy. The new snapshot inherits the
-source `VolumeID`. `register-image`/`copy-image` accept `uefi-preferred` as
-input but treat it as `uefi` at launch.
+`copy-image` is metadata-only — no block copy. The new snapshot inherits the. source `VolumeID`.

 ### EC2 — Volumes (EBS)

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
 | `describe-volumes` | `--volume-ids`, `--filters` (volume-id, status, size, volume-type, attachment.instance-id, attachment.status, attachment.device, availability-zone, tag:*), persisted `DeleteOnTermination` | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
-| `create-volume` | `--size`, `--availability-zone`, `--volume-type` (gp3), `--snapshot-id` | `--iops` (hardcoded 3000), `--encrypted` (hardcoded false), `--throughput`, `--tag-specifications` | **DONE** |
+| `create-volume` | `--size`, `--availability-zone`, `--volume-type` (gp3), `--snapshot-id`, `--tag-specifications` | `--iops` (hardcoded 3000), `--encrypted` (hardcoded false), `--throughput` | **DONE** |
 | `delete-volume` | `--volume-id` | `--dry-run` | **DONE** |
 | `modify-volume` | `--volume-id`, `--size`, `--volume-type`, `--iops` | `--throughput`, `--dry-run`, `--multi-attach-enabled` | **DONE** |
 | `attach-volume` | `--volume-id`, `--instance-id`, `--device` (auto-assigns `/dev/sd[f-p]`) | `--dry-run` | **DONE** |
@@ -228,8 +286,7 @@ input but treat it as `uefi` at launch.

 ### EC2 — Account Settings

-Persistence works (NATS JetStream KV) but stored values are not yet enforced
-by downstream services.
+Persistence works but stored values are not yet enforced by downstream services.

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
@@ -248,10 +305,7 @@ by downstream services.
 | `modify-instance-metadata-defaults` | — | `--http-tokens`, `--http-put-response-hop-limit`, `--http-endpoint`, `--instance-metadata-tags` | **NOT STARTED** |
 | `get-instance-metadata-defaults` | — | `--dry-run` | **NOT STARTED** |

-The account-level `modify-/get-instance-metadata-defaults` are blocked on an
-`aws-sdk-go` bump: the `InstanceMetadataDefaultsResponse` type is absent from the
-pinned v1.44.266 (these APIs landed ~March 2024), and the gateway's generic
-handler needs the typed input struct to route them.
+The account-level `modify-/get-instance-metadata-defaults` are blocked on an `aws-sdk-go` bump: the `InstanceMetadataDefaultsResponse` type is absent from our version (these APIs landed ~March 2024), and the gateway's generic handler needs the typed input struct to route them.

 ### EC2 — VPC Core

@@ -296,8 +350,7 @@ handler needs the typed input struct to route them.

 ### EC2 — Egress-Only Internet Gateway

-KV CRUD only — no OVN/OVS integration. EIGWs are stored but have no effect on
-network topology.
+KV CRUD only — no OVN/OVS integration. EIGWs are stored but have no effect on network topology.

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
@@ -309,7 +362,7 @@ network topology.

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-route-table` | `--vpc-id` | `--tag-specifications`, `--dry-run` | **DONE** |
+| `create-route-table` | `--vpc-id`, `--tag-specifications` | `--dry-run` | **DONE** |
 | `delete-route-table` | `--route-table-id` | `--dry-run` | **DONE** |
 | `describe-route-tables` | `--route-table-ids`, `--filters` (vpc-id, route-table-id, association.main, association.route-table-association-id, association.subnet-id, route.destination-cidr-block, route.gateway-id) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
 | `create-route` | `--route-table-id`, `--destination-cidr-block`, `--gateway-id`, `--nat-gateway-id` | `--egress-only-internet-gateway-id`, `--vpc-peering-connection-id`, `--dry-run` | **DONE** |
@@ -337,7 +390,7 @@ termination. Standalone attach/detach API is internal-only.

 ### EC2 — Elastic IP

-EIP commands are only registered when an external IPAM pool is configured.
+EIP handlers are always registered. Without a public IPAM pool (external mode disabled, or `nat` without a public pool), `describe-addresses` returns an empty list and mutating commands return `UnsupportedOperation`. In `nat` mode a public pool (`--external-pool` or `--external-source=dhcp` at init) enables the full EIP surface with host-delivered ingress.

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
@@ -352,7 +405,7 @@ EIP commands are only registered when an external IPAM pool is configured.

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-nat-gateway` | `--subnet-id`, `--allocation-id` | `--connectivity-type`, `--tag-specifications`, `--dry-run` | **DONE** |
+| `create-nat-gateway` | `--subnet-id`, `--allocation-id`, `--tag-specifications` | `--connectivity-type`, `--dry-run` | **DONE** |
 | `delete-nat-gateway` | `--nat-gateway-id` | `--dry-run` | **DONE** |
 | `describe-nat-gateways` | `--nat-gateway-ids`, `--filters` (vpc-id, state) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
 | `assign-private-nat-gateway-address` | — | `--nat-gateway-id`, `--private-ip-addresses` | **NOT STARTED** |
@@ -362,9 +415,9 @@ EIP commands are only registered when an external IPAM pool is configured.

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-placement-group` | `--group-name`, `--strategy` (spread/cluster) | `--partition-count`, `--spread-level`, `--tag-specifications`, `--dry-run` | **DONE** |
+| `create-placement-group` | `--group-name`, `--strategy` (spread/cluster), `--tag-specifications` | `--partition-count`, `--spread-level`, `--dry-run` | **DONE** |
 | `delete-placement-group` | `--group-name` | `--dry-run` | **DONE** |
-| `describe-placement-groups` | `--group-names`, `--group-ids`, `--filters` (strategy, state, spread-level, group-name) | `--dry-run` | **DONE** |
+| `describe-placement-groups` | `--group-names`, `--group-ids`, `--filters` (strategy, state, spread-level, group-name, tag:*, tag-key, tag-value) | `--dry-run` | **DONE** |

 ### EC2 — VPC Peering

@@ -432,11 +485,13 @@ EIP commands are only registered when an external IPAM pool is configured.

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-launch-template` | — | `--launch-template-name`, `--launch-template-data`, `--tag-specifications` | **NOT STARTED** |
-| `create-launch-template-version` | — | `--launch-template-id`/`--launch-template-name`, `--launch-template-data`, `--source-version` | **NOT STARTED** |
-| `delete-launch-template` | — | `--launch-template-id`/`--launch-template-name`, `--dry-run` | **NOT STARTED** |
-| `describe-launch-templates` | — | `--launch-template-ids`, `--launch-template-names`, `--filters` | **NOT STARTED** |
-| `describe-launch-template-versions` | — | `--launch-template-id`/`--launch-template-name`, `--versions`, `--min-version`, `--max-version` | **NOT STARTED** |
+| `create-launch-template` | `--launch-template-name`, `--launch-template-data` (full nested RequestLaunchTemplateData), `--version-description`, `--tag-specifications` (launch-template-scoped), `--dry-run` (no-op) | `--client-token` (idempotency) | **DONE** |
+| `create-launch-template-version` | `--launch-template-id`/`--launch-template-name`, `--launch-template-data`, `--source-version` (clone-and-override), `--version-description`, `--dry-run` (no-op) | `--client-token`, `--resolve-alias` | **DONE** |
+| `delete-launch-template` | `--launch-template-id`/`--launch-template-name`, `--dry-run` (no-op) | — | **DONE** |
+| `delete-launch-template-versions` | `--launch-template-id`/`--launch-template-name`, `--versions` (rejects the current default version), `--dry-run` (no-op) | — | **DONE** |
+| `modify-launch-template` | `--launch-template-id`/`--launch-template-name`, `--default-version`, `--dry-run` (no-op) | — | **DONE** |
+| `describe-launch-templates` | `--launch-template-ids`, `--launch-template-names`, `--filters` (launch-template-id, launch-template-name, create-time, tag:*, tag-key) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
+| `describe-launch-template-versions` | `--launch-template-id`/`--launch-template-name`, `--versions` (`$Default`/`$Latest`/numeric), `--min-version`, `--max-version`, `--filters` (is-default-version, image-id, instance-type, kernel-id, ram-disk-id, ebs-optimized) | `--max-results`, `--next-token`, `--dry-run`, `--resolve-alias` | **DONE** |

 ### EC2 — Dedicated Hosts, IPv4 Pools, DHCP, Capacity Reservations

@@ -480,8 +535,7 @@ EIP commands are only registered when an external IPAM pool is configured.

 ## IAM

-All IAM operations are account-scoped. Root user (account `000000000000`)
-bypasses policy evaluation entirely.
+All IAM operations are account-scoped. Root user (account `000000000000`) bypasses policy evaluation entirely.

 ### IAM — Users

@@ -491,6 +545,21 @@ bypasses policy evaluation entirely.
 | `get-user` | `--user-name` | — | **DONE** |
 | `list-users` | `--path-prefix` | `--max-items`, `--marker` | **DONE** |
 | `delete-user` | `--user-name` | — | **DONE** |
+| `update-user` | — | `--user-name`, `--new-path`, `--new-user-name` | **NOT STARTED** |
+| `put-user-policy` | `--user-name`, `--policy-name`, `--policy-document` | — | **DONE** |
+| `get-user-policy` | `--user-name`, `--policy-name` | — | **DONE** |
+| `delete-user-policy` | `--user-name`, `--policy-name` | — | **DONE** |
+| `list-user-policies` | `--user-name` | `--max-items`, `--marker` | **DONE** |
+| `tag-user` | `--user-name`, `--tags` | — | **DONE** |
+| `untag-user` | `--user-name`, `--tag-keys` | — | **DONE** |
+| `list-user-tags` | `--user-name` | `--max-items`, `--marker` | **DONE** |
+| `put-user-permissions-boundary` | — | `--user-name`, `--permissions-boundary` | **NOT STARTED** |
+| `delete-user-permissions-boundary` | — | `--user-name` | **NOT STARTED** |
+| `create-login-profile` | — | `--user-name`, `--password` | **NOT STARTED** |
+| `get-login-profile` | — | `--user-name` | **NOT STARTED** |
+| `update-login-profile` | — | `--user-name`, `--password` | **NOT STARTED** |
+| `delete-login-profile` | — | `--user-name` | **NOT STARTED** |
+| `change-password` | — | `--old-password`, `--new-password` | **NOT STARTED** |

 ### IAM — Access Keys

@@ -500,6 +569,7 @@ bypasses policy evaluation entirely.
 | `list-access-keys` | `--user-name` | `--max-items`, `--marker` | **DONE** |
 | `delete-access-key` | `--access-key-id`, `--user-name` | — | **DONE** |
 | `update-access-key` | `--access-key-id`, `--user-name`, `--status` (Active/Inactive) | — | **DONE** |
+| `get-access-key-last-used` | — | `--access-key-id` | **NOT STARTED** |

 ### IAM — Policies

@@ -514,6 +584,17 @@ bypasses policy evaluation entirely.
 | `attach-user-policy` | `--user-name`, `--policy-arn` | — | **DONE** |
 | `detach-user-policy` | `--user-name`, `--policy-arn` | — | **DONE** |
 | `list-attached-user-policies` | `--user-name` | `--path-prefix`, `--max-items`, `--marker` | **DONE** |
+| `create-policy-version` | — | `--policy-arn`, `--policy-document`, `--set-as-default` | **NOT STARTED** |
+| `delete-policy-version` | — | `--policy-arn`, `--version-id` | **NOT STARTED** |
+| `set-default-policy-version` | — | `--policy-arn`, `--version-id` | **NOT STARTED** |
+| `list-entities-for-policy` | — | `--policy-arn`, `--entity-filter`, `--path-prefix`, `--policy-usage-filter` | **NOT STARTED** |
+| `tag-policy` | `--policy-arn`, `--tags` | — | **DONE** |
+| `untag-policy` | `--policy-arn`, `--tag-keys` | — | **DONE** |
+| `list-policy-tags` | `--policy-arn` | `--max-items`, `--marker` | **DONE** |
+| `generate-service-last-accessed-details` | — | `--arn`, `--granularity` | **NOT STARTED** |
+| `get-service-last-accessed-details` | — | `--job-id` | **NOT STARTED** |
+| `get-service-last-accessed-details-with-entities` | — | `--job-id`, `--service-namespace` | **NOT STARTED** |
+| `list-policies-granting-service-access` | — | `--arn`, `--service-namespaces` | **NOT STARTED** |

 ### IAM — Roles

@@ -532,6 +613,12 @@ bypasses policy evaluation entirely.
 | `put-role-policy` | `--role-name`, `--policy-name`, `--policy-document` | — | **DONE** |
 | `get-role-policy` | `--role-name`, `--policy-name` | — | **DONE** (document returned as raw JSON, not URL-encoded) |
 | `delete-role-policy` | `--role-name`, `--policy-name` | — | **DONE** |
+| `put-role-permissions-boundary` | — | `--role-name`, `--permissions-boundary` | **NOT STARTED** |
+| `delete-role-permissions-boundary` | — | `--role-name` | **NOT STARTED** |
+| `tag-role` | `--role-name`, `--tags` | — | **DONE** |
+| `untag-role` | `--role-name`, `--tag-keys` | — | **DONE** |
+| `list-role-tags` | `--role-name` | `--max-items`, `--marker` | **DONE** |
+| `update-role-description` | — | `--role-name`, `--description` | **NOT STARTED** |

 ### IAM — Instance Profiles

@@ -544,6 +631,9 @@ bypasses policy evaluation entirely.
 | `delete-instance-profile` | `--instance-profile-name` | — | **DONE** |
 | `add-role-to-instance-profile` | `--instance-profile-name`, `--role-name` | — | **DONE** |
 | `remove-role-from-instance-profile` | `--instance-profile-name`, `--role-name` | — | **DONE** |
+| `tag-instance-profile` | `--instance-profile-name`, `--tags` | — | **DONE** |
+| `untag-instance-profile` | `--instance-profile-name`, `--tag-keys` | — | **DONE** |
+| `list-instance-profile-tags` | `--instance-profile-name` | `--max-items`, `--marker` | **DONE** |

 ### IAM — OIDC Providers

@@ -556,22 +646,34 @@ bypasses policy evaluation entirely.
 | `add-client-id-to-open-id-connect-provider` | — | `--open-id-connect-provider-arn`, `--client-id` | **NOT STARTED** |
 | `remove-client-id-from-open-id-connect-provider` | — | `--open-id-connect-provider-arn`, `--client-id` | **NOT STARTED** |
 | `update-open-id-connect-provider-thumbprint` | — | `--open-id-connect-provider-arn`, `--thumbprint-list` | **NOT STARTED** |
-| `tag-open-id-connect-provider` / `untag-open-id-connect-provider` | — | `--open-id-connect-provider-arn`, `--tags`/`--tag-keys` | **NOT STARTED** |
+| `tag-open-id-connect-provider` / `untag-open-id-connect-provider` | `--open-id-connect-provider-arn`, `--tags`/`--tag-keys` | — | **DONE** |
+| `list-open-id-connect-provider-tags` | `--open-id-connect-provider-arn` | `--max-items`, `--marker` | **DONE** |

 ### IAM — Groups

 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-group` | — | `--group-name`, `--path` | **NOT STARTED** |
-| `get-group` | — | `--group-name` | **NOT STARTED** |
-| `list-groups` | — | `--path-prefix`, `--max-items`, `--marker` | **NOT STARTED** |
-| `delete-group` | — | `--group-name` | **NOT STARTED** |
-| `add-user-to-group` | — | `--group-name`, `--user-name` | **NOT STARTED** |
-| `remove-user-from-group` | — | `--group-name`, `--user-name` | **NOT STARTED** |
-| `list-groups-for-user` | — | `--user-name`, `--max-items`, `--marker` | **NOT STARTED** |
-| `attach-group-policy` | — | `--group-name`, `--policy-arn` | **NOT STARTED** |
-| `detach-group-policy` | — | `--group-name`, `--policy-arn` | **NOT STARTED** |
-| `list-attached-group-policies` | — | `--group-name`, `--path-prefix`, `--max-items`, `--marker` | **NOT STARTED** |
+| `create-group` | `--group-name`, `--path` | — | **DONE** |
+| `get-group` | `--group-name` | — | **DONE** |
+| `list-groups` | `--path-prefix` | `--max-items`, `--marker` | **DONE** |
+| `delete-group` | `--group-name` | — | **DONE** |
+| `update-group` | — | `--group-name`, `--new-path`, `--new-group-name` | **NOT STARTED** |
+| `add-user-to-group` | `--group-name`, `--user-name` | — | **DONE** |
+| `remove-user-from-group` | `--group-name`, `--user-name` | — | **DONE** |
+| `list-groups-for-user` | `--user-name` | `--max-items`, `--marker` | **DONE** |
+| `attach-group-policy` | `--group-name`, `--policy-arn` | — | **DONE** |
+| `detach-group-policy` | `--group-name`, `--policy-arn` | — | **DONE** |
+| `list-attached-group-policies` | `--group-name`, `--path-prefix` | `--max-items`, `--marker` | **DONE** |
+| `put-group-policy` | `--group-name`, `--policy-name`, `--policy-document` | — | **DONE** |
+| `get-group-policy` | `--group-name`, `--policy-name` | — | **DONE** |
+| `delete-group-policy` | `--group-name`, `--policy-name` | — | **DONE** |
+| `list-group-policies` | `--group-name` | `--max-items`, `--marker` | **DONE** |
+
+### IAM — Account
+
+| Command | Implemented Flags | Missing Flags | Status |
+|---------|-------------------|---------------|--------|
+| `get-account-summary` | — | — | **DONE** |

 ---

@@ -588,7 +690,7 @@ bypasses policy evaluation entirely.
 | `get-federation-token` | — | `--name`, `--policy`, `--policy-arns`, `--duration-seconds`, `--tags` | **NOT STARTED** |
 | `decode-authorization-message` | — | `--encoded-message` | **NOT STARTED** |

-Trust policies stored on roles (`AssumeRolePolicyDocument`) reject `Condition`, `NotPrincipal`, `NotAction`, empty-string `Action` elements, and empty `Principal` blocks at write time (`MalformedPolicyDocument`) — v1 has no Condition evaluator so accepting them would silently allow.
+Trust policies (`AssumeRolePolicyDocument`) reject `NotPrincipal`, `NotAction`, empty-string `Action` elements, and empty `Principal` blocks at write time (`MalformedPolicyDocument`). `Condition` blocks are rejected except on `sts:AssumeRoleWithWebIdentity` with `StringEquals` (IRSA), which v1 evaluates at assume time (`{iss}:sub`, `{iss}:aud`); anything wider is rejected to avoid silent over-grant.

 ---

@@ -625,7 +727,7 @@ curl -i http://169.254.169.254/latest/meta-data/instance-id
 | `/latest/meta-data/ami-id` | GET | launch `ImageId` | **DONE** |
 | `/latest/meta-data/ami-launch-index` | GET | Per-instance launch index (`0..n-1`), contiguous on partial failure | **DONE** |
 | `/latest/meta-data/reservation-id` | GET | `DescribeInstances` `Reservation.ReservationId` | **DONE** |
-| `/latest/meta-data/instance-life-cycle` | GET | `on-demand` (Spot not modelled) | **DONE** |
+| `/latest/meta-data/instance-life-cycle` | GET | `spot` for a spot-launched instance, else `on-demand`; defaults to `on-demand` on a resolution miss (never 404, the leaf is always advertised) | **DONE** |
 | `/latest/meta-data/local-ipv4` | GET | `ENIRecord.PrivateIpAddress` (== request source IP) | **DONE** |
 | `/latest/meta-data/public-ipv4` | GET | EIP, else instance public IP; empty body if none | **DONE** |
 | `/latest/meta-data/public-hostname` | GET | Mirrors `public-ipv4`; 404 when no public IP | **DONE** |
@@ -651,7 +753,7 @@ curl -i http://169.254.169.254/latest/meta-data/instance-id
 | `/latest/meta-data/block-device-mapping/...` | GET | `ami`/`root`/`ebsN`/`ephemeralN` device map | **NOT STARTED** (404) |
 | `/latest/meta-data/placement/{group-name,partition-number,availability-zone-id,host-id}` | GET | Placement extras beyond `availability-zone`/`region` | **NOT STARTED** (404) |
 | `/latest/meta-data/instance-action` | GET | `none` unless interruptible instances ship | **NOT STARTED** (404) |
-| `/latest/meta-data/spot/{instance-action,termination-time}` | GET | Only meaningful once Spot is modelled | **NOT STARTED** (404) |
+| `/latest/meta-data/spot/{instance-action,termination-time}` | GET | 404 is the faithful steady state for a never-interrupted spot instance ("no action scheduled"); a 200 body would trigger interruption handling in pollers (AWS Node Termination Handler / Karpenter). Not advertised in the `spot/` listing | **DONE** (404 by contract) |

 ---

@@ -49,7 +49,7 @@ Request routing (`spinifex/gateway/gateway.go`):

 1. **Authentication**: SigV4 middleware validates AWS credentials and resolves the account ID
 2. **Throttling**: Per-account+action token bucket rejects bursts post-auth
-3. **Service Detection**: Extracts service name (ec2, iam, account, elasticloadbalancing, spinifex) from the Authorization header
+3. **Service Detection**: Extracts service name from the Authorization header
 4. **Action Dispatch**: Routes to service-specific handler

 ```go
@@ -60,8 +60,20 @@ case "account":
    err = gw.Account_Request(w, r)
 case "iam":
    err = gw.IAM_Request(w, r)
+case "sts":
+    err = gw.STS_Request(w, r)
 case "elasticloadbalancing":
    err = gw.ELBv2_Request(w, r)
+case "eks":
+    err = gw.EKS_Request(w, r)
+case "ecs":
+    err = gw.ECS_Request(w, r)
+case "ecr":
+    err = gw.ECR_Request(w, r)
+case "acm":
+    err = gw.ACM_Request(w, r)
+case "tagging":
+    err = gw.Tagging_Request(w, r)
 case "spinifex":
    err = gw.Spinifex_Request(w, r)
 }
@@ -79,18 +91,12 @@ The EC2 handler (`spinifex/gateway/ec2.go`) parses the `Action` parameter and de
    return gateway_ec2_instance.DescribeInstances(input, gw.NATSConn, gw.DiscoverActiveNodes(), accountID)
 }),
 // ... + volumes, snapshots, VPCs, subnets, route tables, IGWs, NAT gateways,
-// security groups, network interfaces, placement groups, key pairs, images, tags
+// security groups, network interfaces, elastic IPs, placement groups, key pairs,
+// images, tags, capacity reservations, spot instances
 ```

 ### 4. NATS Messaging

-The full service-to-subject map — which service subscribes to what, what
-each one publishes downstream, what each depends on — is captured in
-[`service-interfaces.yaml`](./service-interfaces.yaml). It is the source
-of truth for targeted e2e suite selection (see
-`docs/development/improvements/e2e-targeted-suite-selection.md`) and is
-validated by `make manifest-check`.
-
 The gateway communicates with daemons via NATS request/response. Most calls go through `utils.NATSRequest`, which marshals the input, attaches the account ID as a NATS header, and unmarshals the typed response:

 ```go
@@ -114,7 +120,7 @@ Daemons (`spinifex/daemon/daemon.go`) subscribe to NATS topics and handle reques
 {"ec2.DescribeInstanceTypes", d.handleEC2DescribeInstanceTypes, ""},
 ```

-`ec2.RunInstances` is dynamic: `ResourceManager` subscribes to `ec2.RunInstances.{type}` (and `ec2.RunInstances.{type}.{nodeId}` for targeted dispatch) only while the node has capacity for that type, and unsubscribes when full. Per-instance commands flow over `ec2.cmd.{instanceID}`, subscribed only by the owning node.
+`ec2.RunInstances` is dynamic: `ResourceManager` subscribes to `ec2.RunInstances.{type}` only while the node has capacity for that type, and unsubscribes when full. The node-targeted subject `ec2.RunInstances.{type}.{nodeId}` stays subscribed even at full capacity — the launch-time `allocate()` enforces limits there. Per-instance commands flow over `ec2.cmd.{instanceID}`, subscribed only by the owning node.

 **Queue Groups**: topics with the `spinifex-workers` queue group are load-balanced — only one daemon handles each request. Topics without a queue group fan out to all daemons.

@@ -151,7 +157,7 @@ type Daemon struct {
    config        *config.Config          // Node-specific configuration
    natsConn      *nats.Conn              // NATS connection
    resourceMgr   *ResourceManager        // CPU/Memory tracking + dynamic RunInstances subscriptions
-    Instances     vm.Instances            // Local VMs
+    vmMgr         *vm.Manager             // Local VMs
    // ... + one service struct per EC2/ELBv2 resource
    //       (instance, key, image, volume, snapshot, tags, vpc, subnet,
    //        igw, eigw, natgw, routetable, eip, placementgroup, elbv2, ...)
@@ -172,6 +178,8 @@ type ResourceManager struct {
    allocatedVCPU int
    allocatedMem  float64
    instanceTypes map[string]*ec2.InstanceTypeInfo   // t3.micro, t3.small, etc.
+    // ... + capacity-reservation accounting, GPU manager, nbdkit memory
+    //       overhead tracking, and the dynamic-subscription state
 }
 ```

@@ -236,22 +244,26 @@ Cluster configuration (`spinifex/config/config.go`):

 ```go
 type ClusterConfig struct {
-    Epoch   uint64            // Incremented on config changes
-    Node    string            // This node's identifier
-    Version string            // Spinifex version
-    Nodes   map[string]Config // Configuration for all nodes
+    Epoch     uint64            // Incremented on config changes
+    Node      string            // This node's identifier
+    Version   string            // Spinifex version
+    Network   NetworkConfig     // Cluster-wide external network settings (IP pools, IPsec)
+    Bootstrap BootstrapConfig   // Default VPC IDs written by admin init
+    AWS       AWSConfig         // Cluster-wide AWS-parity settings (region, endpoint suffix)
+    Nodes     map[string]Config // Configuration for all nodes
 }

 type Config struct {
-    Node, Host, Region, AZ string
-    BaseDir, DataDir       string
+    Node, Host, AdvertiseIP  string
+    Region, AZ               string
+    BaseDir, DataDir, WalDir string
+    Services   []string         // Which services this node runs locally
    Daemon     DaemonConfig
    NATS       NATSConfig       // Broker address, JetStream, ACL token, CA cert
    AWSGW      AWSGWConfig      // Gateway host, TLS certs, throttle config
    Predastore PredastoreConfig // S3 endpoint + service credentials
    Viperblock ViperblockConfig
    VPCD       VPCDConfig       // OVN/OVS endpoints
-    Network    NetworkConfig
 }
 ```

@@ -263,14 +275,14 @@ Configs are generated by `spx admin init` (leader) or `spx admin join` (follower
 spinifex/
 ├── spinifex/
 │   ├── services/                # Long-running services (awsgw, spinifex, viperblockd,
-│   │                            #   predastore, nats, vpcd, spinifexui)
+│   │                            #   predastore, nats, spinifexui)
 │   ├── gateway/                 # AWS API surface
 │   │   ├── gateway.go           # chi router, SigV4 auth, throttling
 │   │   ├── auth.go              # SigV4 verification + IAM lookup
 │   │   ├── ec2.go               # EC2 action dispatcher
 │   │   ├── ec2/                 # Per-resource handlers (instance, volume, vpc, ...)
 │   │   ├── elbv2/, iam/, policy/
-│   ├── handlers/                # NATS-side service implementations (ec2, elbv2, iam)
+│   ├── handlers/                # NATS-side service implementations (ec2, elbv2, iam, ...)
 │   ├── daemon/
 │   │   ├── daemon.go            # Daemon entry, NATS subscription table
 │   │   └── daemon_handlers_*.go # One file per resource family
@@ -279,11 +291,13 @@ spinifex/
 │   ├── instancetypes/           # t3.*, m5.*, sys.* definitions
 │   ├── lbagent/                 # In-VM agent for ELB target health
 │   ├── vm/                      # VM instance representation
+│   ├── vpcd/                    # VPC daemon (OVN/OVS network programming)
 │   └── qmp/                     # QEMU Machine Protocol client
 └── cmd/
    ├── spinifex/                # spx CLI
    ├── lb-agent/                # ELB target agent binary
-    └── installer/               # Platform installer
+    ├── installer/               # Platform installer
+    └── ...                      # ECS agent, ECR credential provider, EKS helper binaries
 ```

 ## Development
@@ -1,11 +1,7 @@
 # Spinifex Tag Conventions

-Spinifex uses a small set of reserved tag keys to mark resources that are
-owned and managed by the platform itself (as opposed to resources created
-directly by customer AWS API calls). The UI uses these tags to hide
-system-owned resources from customer-facing listings so operators do not
-mistake them for user workloads and accidentally terminate or modify
-them.
+Spinifex uses a small set of reserved tag keys to mark resources that are owned and managed by the platform itself (as opposed to resources created directly by customer AWS API calls). The UI uses these tags to hide system-owned resources from customer-facing listings so operators do not
+mistake them for user workloads and accidentally terminate or modify them.

 ## `spinifex:managed-by`

@@ -17,50 +13,20 @@ Identifies the Spinifex platform component that owns a resource.

 The tag is applied by the backend at create/launch time:

- **Instance** — set in
-  `spinifex/spinifex/daemon/daemon_system_instance.go` when
-  `LaunchSystemInstance` builds the `RunInstancesInput`, and mirrored onto
-  the stored `vm.VM.ManagedBy` field for fast lookup.
- **ENI** — set in
-  `spinifex/spinifex/handlers/elbv2/service_impl.go` when
-  `CreateLoadBalancer` calls `CreateNetworkInterface`.
- **AMI** — declared on the catalog entry in
-  `spinifex/spinifex/utils/images.go` and copied into
-  `AMIMetadata.Tags` by the image importer in
-  `spinifex/cmd/spinifex/cmd/admin.go`.
+- **Instance** — set in `spinifex/spinifex/daemon/daemon_system_instance.go` when `LaunchSystemInstance` builds the `RunInstancesInput`, and mirrored onto the stored `vm.VM.ManagedBy` field for fast lookup.
+- **ENI** — set in `spinifex/spinifex/handlers/elbv2/service_impl.go` when `CreateLoadBalancer` calls `CreateNetworkInterface`.
+- **AMI** — declared on the catalog entry in `spinifex/spinifex/utils/images.go` and copied into `AMIMetadata.Tags` by the image importer in `spinifex/cmd/spinifex/cmd/admin.go`.

-Tag keys and values are defined once in `spinifex/spinifex/tags/tags.go`;
-new system components should register their value there and reuse the
-constants rather than hardcoding strings.
+Tag keys and values are defined once in `spinifex/spinifex/tags/tags.go`; new system components should register their value there and reuse the constants rather than hardcoding strings.

 ## UI behaviour

-The admin **Nodes** page and the **Images** (AMIs) page filter out
-resources that carry `spinifex:managed-by`. Engineers who need to see
-system resources for debugging can append `?system=1` to the URL — the
-filter is bypassed and a hidden-count banner is replaced with the full
-listing. Direct navigation to a system AMI detail page without
-`?system=1` renders the "Image not found" fallback.
+The admin **Nodes** page and the **Images** (AMIs) page filter out resources that carry `spinifex:managed-by`. Engineers who need to see system resources for debugging can append `?system=1` to the URL — the filter is bypassed and a hidden-count banner is replaced with the full listing. Direct navigation to a system AMI detail page without `?system=1` renders the "Image not found" fallback.

-The EC2 **Instances** page does not need UI-side filtering: backend
-SigV4 authentication scopes `DescribeInstances` by account ID, and
-system instances live under the global system account
+The EC2 **Instances** page does not need UI-side filtering: backend SigV4 authentication scopes `DescribeInstances` by account ID, and system instances live under the global system account
 (`000000000000`), so customers never see them.

 ## `spinifex:lb-arn`

-Applied to ELBv2-managed ENIs alongside `spinifex:managed-by=elbv2`.
-Stores the ARN of the parent load balancer so reconciliation can map an
-ENI back to its owner without a separate lookup. Customer-facing
-filtering keys off `spinifex:managed-by`; `spinifex:lb-arn` is
-informational.
-
-## Adding a new system component
-
-1. Add a new value constant next to `ManagedByELBv2` in
-   `spinifex/spinifex/tags/tags.go`.
-2. Tag every resource the component creates (instance, ENI, AMI,
-   volume, …) with `spinifex:managed-by=<value>`.
-3. Add the value to the table above so operators can recognise it.
-4. No frontend changes are required — the UI filters on the key, not
-   the value.
+Applied to ELBv2-managed ENIs alongside `spinifex:managed-by=elbv2`. Stores the ARN of the parent load balancer so reconciliation can map an ENI back to its owner without a separate lookup. Customer-facing
+filtering keys off `spinifex:managed-by`; `spinifex:lb-arn` is informational.
@@ -1,7 +1,7 @@
 ---
 title: "Setting Up Your Cluster"
 description: "Import an AMI, create a VPC with a public subnet, and launch your first EC2 instance on Spinifex."
-category: "Getting Started"
+category: "Admin"
 tags:
  - setup
  - vpc
@@ -33,16 +33,14 @@ resources:

 ## Overview

-This guide assumes Spinifex is already installed and running. If not, follow one of the installation guides first:
- [Single-Node Install](/docs/install)
- [Multi-Node Install](/docs/install-multi-node)
- [Source Install](/docs/install-source)
+This guide walks through the first steps on a freshly installed cluster: importing an AMI, creating an SSH key and a VPC with a public subnet, launching an instance, and connecting to it.

 ## Instructions

 ## Prerequisites

-Ensure the AWS profile is set:
+- Spinifex installed and running — see [Single-Node Install](/docs/install), [Multi-Node Install](/docs/install-multi-node), or [Source Install](/docs/install-source)
+- AWS CLI configured with the `spinifex` profile:

 ```bash
 export AWS_PROFILE=spinifex
@@ -1,7 +1,7 @@
 ---
 title: "Spinifex Admin CLI"
 description: "Complete reference for the Spinifex administration CLI. Manage accounts, nodes, VMs, and services."
-category: "Administration"
+category: "Admin"
 tags:
  - cli
  - admin
@@ -25,30 +25,56 @@ resources:

 ## Overview

-The `spx` binary is the central administration tool for managing your Spinifex infrastructure. It provides commands for cluster initialization, account management, node operations, VM lifecycle, and service control.
+The `spx` binary is the central administration tool for managing your Spinifex infrastructure. It provides commands for cluster initialization, account management, node operations, VM lifecycle, and service control. All services in the Spinifex platform are managed through this single binary.

-**Binary location:** `/usr/local/bin/spx` (binary install) or `~/Development/mulga/spinifex/bin/spx` (source install)
-
-All services in the Spinifex platform are managed through this single binary.
+**Binary location:** `/usr/local/bin/spx`

 ## Instructions

 ## Account Management

-Create a new isolated account with an admin user and AdministratorAccess policy:
+Create a new isolated account. This provisions a sequential 12-digit account ID, an `admin` user with an `AdministratorAccess` policy attached, and an access key pair. The credentials are written to `~/.aws/credentials` and `~/.aws/config` under a `spinifex-<name>` profile automatically.

 ```bash
 spx admin account create --name myteam
+```
+
+```
+Account created successfully!
+  Account ID:        000000000002
+  Account Name:      myteam
+  Admin User:        admin
+  Access Key ID:     AKIA1A2B3C4D5E6F7890ABCD
+  Secret Access Key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+  AWS Profile:       spinifex-myteam
+
+Use with:
+  AWS_PROFILE=spinifex-myteam aws ec2 describe-instances
+```
+
+> **The secret access key is only shown once.** It is saved to `~/.aws/credentials` on the node where you ran the command; copy it from there if you need it elsewhere.
+
+Set the profile to start using the account:
+
+```bash
 export AWS_PROFILE=spinifex-myteam
 ```

+To create additional users and scoped permissions within the account, see [IAM Users and Policies](/docs/iam-users-and-policies).
+
 List all accounts:

 ```bash
 spx admin account list
 ```

-
+```
+ACCOUNT ID     NAME                 STATUS     CREATED
+----------     ----                 ------     -------
+000000000000   system               ACTIVE     2026-07-03 02:17
+000000000001   spinifex             ACTIVE     2026-07-03 02:17
+000000000002   myteam               ACTIVE     2026-07-03 03:32
+```

 ## Node Management

@@ -97,7 +123,7 @@ configuration is needed.

 ## Cluster Shutdown

-Coordinated, phased shutdown of the entire cluster (API/UI → VMs → viperblock → predastore → NATS/daemon):
+Coordinated, phased shutdown of the entire cluster — running VMs are stopped before storage and control-plane services:

 ```bash
 spx admin cluster shutdown
@@ -1,7 +1,7 @@
 ---
 title: "Updating Spinifex"
 description: "Upgrade an existing Spinifex installation to the latest release."
-category: "Administration"
+category: "Admin"
 tags:
  - update
  - upgrade
@@ -1,55 +0,0 @@
---
-title: "Creating Accounts"
-description: "Create and manage isolated user accounts with their own resources."
-category: "Administration"
-tags:
-  - accounts
-  - iam
-  - admin
-resources:
-  - title: "Spinifex Repository"
-    url: "https://github.com/mulgadc/spinifex"
---
-
-# Creating Accounts
-
-> Create and manage isolated user accounts with their own resources.
-
-## Table of Contents
-
- [Overview](#overview)
- [Instructions](#instructions)
- [Troubleshooting](#troubleshooting)
-
---
-
-## Overview
-
-Spinifex supports multi-tenant account isolation. Each account gets its own IAM credentials, AWS CLI profile, and isolated resource namespace.
-
-## Instructions
-
-## Create Account
-
-```bash
-spx admin account create --name myteam
-export AWS_PROFILE=spinifex-myteam
-```
-
-## Troubleshooting
-
-## Credentials Not Working
-
-Verify the AWS CLI configuration files exist and contain the correct profile:
-
-```bash
-cat ~/.aws/config
-cat ~/.aws/credentials
-```
-
-Ensure the `AWS_PROFILE` environment variable matches the profile name:
-
-```bash
-echo $AWS_PROFILE
-export AWS_PROFILE=spinifex-myteam
-```
@@ -72,7 +72,7 @@ After setup and reboot, enable GPU passthrough:
 sudo spx admin gpu enable
 ```

-The command writes the configuration, signals the daemon (`SIGHUP`), and waits up to 30 seconds for the daemon to confirm the new state. On success it prints the current GPU status.
+The command writes the configuration, notifies the daemon and waits up to 30 seconds for the daemon to confirm the new state. On success it prints the current GPU status.

 <img src="../../../.github/assets/images/gpu-enabled.png" alt="GPU enabled">

@@ -22,11 +22,12 @@ resources:
 - [Overview](#overview)
 - [Launch](#launch)
 - [Manage](#manage)
- [Reboot](#reboot)
 - [Modify Instance Attributes](#modify-instance-attributes)
+- [Instance Metadata Options](#instance-metadata-options)
+- [Spot Instances](#spot-instances)
 - [Console Output](#console-output)
 - [Instance Types](#instance-types)
- [SSH (Development)](#ssh-development)
+- [SSH](#ssh)
 - [Troubleshooting](#troubleshooting)
  - [Instance Fails to Boot](#instance-fails-to-boot)
  - [Cannot SSH Into Instance](#cannot-ssh-into-instance)
@@ -37,22 +38,12 @@ resources:

 Spinifex provides EC2-compatible VM management built on QEMU/KVM. Instances support cloud-init, SSH key injection, VPC networking, and standard AWS lifecycle operations.

-**Supported operations:**
-
- `run-instances` — Launch new VMs
- `describe-instances` — Query state
- `stop-instances` / `start-instances` — Lifecycle
- `reboot-instances` — In-place restart (QMP reset)
- `terminate-instances` — Permanent removal
- `modify-instance-attribute` — Change type, user data, or source/dest check
- `get-console-output` — Retrieve serial console log
- `describe-instance-types` — List available instance types
-
 ## Instructions

 ## Prerequisites

-Ensure the AWS profile is set:
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:

 ```bash
 export AWS_PROFILE=spinifex
@@ -61,18 +52,20 @@ export AWS_PROFILE=spinifex
 ## Launch

 ```bash
-aws ec2 run-instances \
+INSTANCE_ID=$(aws ec2 run-instances \
  --image-id $SPINIFEX_AMI \
  --instance-type t3.small \
-  --key-name spinifex-key
-
-export INSTANCE_ID="i-XXX"
+  --key-name spinifex-key \
+  --query 'Instances[0].InstanceId' --output text)
 ```

+Other supported launch flags: `--subnet-id`, `--security-group-ids`, `--tag-specifications`, `--user-data`, `--iam-instance-profile`, `--block-device-mappings`, `--placement`, `--count`.
+
 ## Manage

 ```bash
 aws ec2 describe-instances --instance-ids $INSTANCE_ID
+aws ec2 describe-instance-status --instance-ids $INSTANCE_ID --include-all-instances
 aws ec2 stop-instances --instance-ids $INSTANCE_ID
 aws ec2 start-instances --instance-ids $INSTANCE_ID
 aws ec2 terminate-instances --instance-ids $INSTANCE_ID
@@ -81,7 +74,7 @@ aws ec2 reboot-instances --instance-ids $INSTANCE_ID

 ## Modify Instance Attributes

-Change instance type, user data, or source/dest check. Instance type and user data require the instance to be **stopped** first.
+Change instance type, user data, or termination protection. Instance type and user data require the instance to be **stopped** first.

 ### Change Instance Type

@@ -95,6 +88,37 @@ aws ec2 modify-instance-attribute \
 aws ec2 start-instances --instance-ids $INSTANCE_ID
 ```

+### Termination Protection
+
+```bash
+aws ec2 modify-instance-attribute \
+  --instance-id $INSTANCE_ID \
+  --disable-api-termination
+```
+
+## Instance Metadata Options
+
+IMDSv2 is always enforced — `--http-tokens optional` and `--http-endpoint disabled` are rejected with `UnsupportedOperation`, matching AWS account-level IMDSv2 enforcement. The hop limit is adjustable (raise it for containerised workloads that reach IMDS through an extra network hop):
+
+```bash
+aws ec2 modify-instance-metadata-options \
+  --instance-id $INSTANCE_ID \
+  --http-put-response-hop-limit 2
+```
+
+## Spot Instances
+
+Spot requests are supported as a compatibility layer over the on-demand path: requests launch real VMs on your own compute immediately and report `active`/`fulfilled`. There is no spot market — no bidding, interruption, or reclamation.
+
+```bash
+aws ec2 request-spot-instances \
+  --instance-count 1 \
+  --launch-specification '{"ImageId":"'$SPINIFEX_AMI'","InstanceType":"t3.small","KeyName":"spinifex-key"}'
+
+aws ec2 describe-spot-instance-requests
+aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $SIR_ID
+```
+
 ## Console Output

 Retrieve the serial console log for a running instance. Output is base64-encoded.
@@ -121,7 +145,8 @@ aws ec2 describe-instance-types
 Filter to a specific type:

 ```bash
-aws ec2 describe-instance-types --instance-types t3.micro t3.small
+aws ec2 describe-instance-types \
+  --query "InstanceTypes[?InstanceType=='t3.small']"
 ```

 Show capacity (how many of each type can still be launched):
@@ -131,16 +156,9 @@ aws ec2 describe-instance-types \
  --filters Name=capacity,Values=true
 ```

-## SSH (Development)
+## SSH

-In development mode, find the QEMU port forward and connect via localhost:
-
-```bash
-ps auxw | grep $INSTANCE_ID
-ssh -i ~/.ssh/spinifex-key ubuntu@127.0.0.1 -p <port>
-```
-
-For production SSH via public IPs, see [Setting Up Your Cluster](/docs/setting-up-your-cluster).
+To SSH into instances via their public IPs, see [Setting Up Your Cluster](/docs/setting-up-your-cluster).

 ## Troubleshooting

@@ -50,7 +50,8 @@ The `partition` strategy is not supported.

 ## Prerequisites

-Ensure the AWS profile is set:
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:

 ```bash
 export AWS_PROFILE=spinifex
@@ -191,11 +191,18 @@ is identical:
 Both support the same AWS features: public subnets, Elastic IPs, security groups,
 DescribeInstances showing public IPs.

-## `nat` — Outbound Only (Simple)
+## `nat` — Shared SNAT (Simple)

-All VMs share a single external IP for outbound SNAT. No public IPs, no Elastic
-IPs, no inbound from WAN. All subnets behave as private subnets with internet
-access.
+All VMs share a single external IP for outbound SNAT. By default there are no
+public IPs, no Elastic IPs, and no inbound from WAN — all subnets behave as
+private subnets with internet access. On routed-NAT nodes, adding a public pool
+restores full public IP parity (see below).
+
+> **Limitation (routed-NAT v1):** System instances (ECS/EKS/load-balancer
+> agents) source egress from ExternalIPAM pool IPs, which do not exist in
+> `external_mode=nat`. Those features require `external_mode=pool`. A v2 will
+> either allocate transit IPs for system instances or reject the feature at the
+> API level in nat mode.

 The `gateway_ip` is the IP that OVN uses for SNAT. You can set it statically or
 use `setup-ovn.sh --dhcp` to obtain one from the router. This is the router's
@@ -215,6 +222,90 @@ gateway_ip = "192.168.1.100"     # Single IP for all VM outbound SNAT
 prefix_len = 24
 ```

+### Host access to instances (jumpbox pattern)
+
+The spinifex host automatically reaches every instance's **private IP**: IGW
+attach installs a host route into OVN (`<vpc-cidr> via <gateway-transit-ip>
+dev spx-nat-host`) and exempts the transit net from SNAT, so replies to
+host-initiated connections come back un-NATted. No per-instance setup.
+
+Security groups still apply and the default SG is closed to the host, same as
+AWS — open SSH/ICMP from the transit net first:
+
+```bash
+aws ec2 authorize-security-group-ingress --group-id $SG \
+  --protocol tcp --port 22 --cidr 100.127.0.0/24
+```
+
+Then use the host as a jumpbox for remote access:
+
+```bash
+ssh -J admin@<spinifex-host> ubuntu@<instance-private-ip>
+```
+
+Extra networks that must reach instances without SNAT (e.g. a management LAN)
+can be added via `[network] nat_exempt_cidrs = ["192.168.50.0/24"]`.
+
+### Public IPs in NAT mode (public pool)
+
+A routed-NAT node (`setup-ovn.sh --nat-uplink` + `spx admin init
+--external-mode=nat`) can carry a public pool alongside the internal
+`nat-transit` pool. With one configured, nat mode behaves like pool mode for
+public IPs: `MapPublicIpOnLaunch` on the default subnet, auto-assigned public
+IPs, and Elastic IPs all work. Spinifex delivers each public IP at the host —
+a `/32` route steers it into OVN and a proxy-ARP neighbor entry answers for it
+on the uplink (L3 only, same MAC, so it works on WiFi and other non-bridgeable
+uplinks).
+
+Static range carved out of the router's DHCP scope:
+
+```bash
+spx admin init --external-mode=nat \
+  --external-pool 192.168.1.150-192.168.1.250 \
+  --external-gateway 192.168.1.1
+```
+
+Or lease public IPs from the upstream router's DHCP:
+
+```bash
+spx admin init --external-mode=nat --external-source=dhcp \
+  --external-bind-bridge wlan0
+```
+
+On WiFi/WWAN uplinks the leases are requested with the interface's own MAC
+(`dhcp_mac = "interface"`, written automatically) and distinguished by DHCP
+client-id. Some routers key leases by MAC and ignore the client-id — Spinifex
+detects this (the router hands the same IP to two client-ids) and fails the
+allocation with advice to switch to a static range.
+
+Resulting config:
+
+```toml
+[network]
+external_mode = "nat"
+bridge_mode   = "nat"
+
+[[network.external_pools]]
+name       = "nat-transit"        # internal transit net (auto-generated)
+gateway    = "100.127.0.1"
+prefix_len = 24
+
+[[network.external_pools]]
+name        = "wan"               # public pool
+range_start = "192.168.1.150"
+range_end   = "192.168.1.250"
+gateway     = "192.168.1.1"
+prefix_len  = 24
+```
+
+**Caveat — reaching an EIP from the spinifex host itself.** Host-sourced
+traffic enters OVN from the transit net, which is exempt from NAT (that is what
+makes the jumpbox pattern work) — so a host connection to an EIP that carries
+the transit source IP would skip DNAT. Spinifex stamps the EIP route with the
+uplink's LAN IP as source to avoid this, but if no uplink address can be
+determined, connect to the instance's **private IP** from the host instead.
+Other machines on the LAN are unaffected.
+
 ## Disabled (Empty/Omitted)

 VPC networking is overlay-only. No external connectivity. Instances can only
@@ -222,19 +313,21 @@ communicate within their VPC.

 ## Mode Comparison

-| Capability                        | `pool` (static) | `pool` (dhcp) | `nat`    | Disabled |
-| --------------------------------- | --------------- | ------------- | -------- | -------- |
-| Outbound internet                 | Yes             | Yes           | Yes      | No       |
-| Inbound from WAN                  | Yes (1:1 NAT)   | Yes (1:1 NAT) | No       | No       |
-| Public subnets                    | Yes             | Yes           | No       | No       |
-| Auto-assign public IPs            | Yes             | Yes           | No       | No       |
-| Elastic IPs                       | Yes             | Yes           | No       | No       |
-| DescribeInstances shows public IP | Yes             | Yes           | No       | No       |
-| Admin must reserve IP range       | Yes             | No            | No       | No       |
-| Needs router DHCP                 | No              | Yes           | Optional | No       |
+| Capability                        | `pool` (static) | `pool` (dhcp) | `nat`             | Disabled |
+| --------------------------------- | --------------- | ------------- | ----------------- | -------- |
+| Outbound internet                 | Yes             | Yes           | Yes               | No       |
+| Host reaches instance private IPs | No              | No            | Yes (routed)      | No       |
+| Inbound from WAN                  | Yes (1:1 NAT)   | Yes (1:1 NAT) | With public pool  | No       |
+| Public subnets                    | Yes             | Yes           | With public pool  | No       |
+| Auto-assign public IPs            | Yes             | Yes           | With public pool  | No       |
+| Elastic IPs                       | Yes             | Yes           | With public pool  | No       |
+| DescribeInstances shows public IP | Yes             | Yes           | With public pool  | No       |
+| Admin must reserve IP range       | Yes             | No            | Only static pool  | No       |
+| Needs router DHCP                 | No              | Yes           | Optional          | No       |

-If you start with `nat` and later need public subnets, switch to `pool` and
-define a range (or use `source = "dhcp"`) — no data migration needed.
+If you start with `nat` and later need public subnets: on a bridgeable uplink
+switch to `pool` and define a range (or use `source = "dhcp"`); on a routed-NAT
+node just add a public pool alongside `nat-transit` — no data migration needed.

 ## Bridge Setup — Physical Network Wiring

@@ -452,7 +545,7 @@ dns_servers = ["8.8.8.8"]           # DNS for VMs (optional)

 | Field         | Required    | Description                                                                                                                                                            |
 | ------------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `name`        | Yes         | Unique pool name. Used as NATS KV key and in `AllocateAddress`.                                                                                                        |
+| `name`        | Yes         | Unique pool name. Referenced by `AllocateAddress` to target a specific pool.                                                                                           |
 | `source`      | No          | IP source: `"static"` (default) uses `range_start`/`range_end`. `"dhcp"` obtains IPs from the router's DHCP server on each VM launch.                                  |
 | `range_start` | Static only | First IP in the range. First IP is reserved for OVN gateway SNAT (unless `gateway_ip` overrides).                                                                      |
 | `range_end`   | Static only | Last IP in the range.                                                                                                                                                  |
@@ -462,6 +555,7 @@ dns_servers = ["8.8.8.8"]           # DNS for VMs (optional)
 | `region`      | No          | Scopes pool to a region. Instances in this region prefer this pool.                                                                                                    |
 | `az`          | No          | Scopes pool to an AZ. More specific than region.                                                                                                                       |
 | `dns_servers` | No          | DNS servers propagated to VMs via OVN DHCP.                                                                                                                            |
+| `gw_lrp_range_start` / `gw_lrp_range_end` | No | Reserve gateway-LRP IPs for per-VPC OVN routers. When unset, the allocator auto-derives the top 16 host IPs of the pool subnet (~15 concurrent VPCs). Widen to raise the concurrent-VPC ceiling — the `nat-transit` pool defaults to `100.127.0.16`-`100.127.0.254` (239 VPCs). Must not overlap `range_start`/`range_end`. |

 ### Why range_start/range_end Instead of CIDR?

@@ -483,13 +577,16 @@ These are different things:

 ```toml
 [nodes.spx1.vpcd]
-ovn_nb_addr        = "tcp:10.1.3.181:6641"   # OVN Northbound DB
-ovn_sb_addr        = "tcp:10.1.3.181:6642"   # OVN Southbound DB
+# Comma-separated list of the OVN NB/SB quorum endpoints (3 DB nodes). vpcd and
+# ovn-controller fail over across them and follow the RAFT leader.
+ovn_nb_addr        = "tcp:10.1.3.181:6641,tcp:10.1.3.182:6641,tcp:10.1.3.183:6641"
+ovn_sb_addr        = "tcp:10.1.3.181:6642,tcp:10.1.3.182:6642,tcp:10.1.3.183:6642"
 external_interface = "br-wan"                 # WAN Linux bridge name
 ```

 | Field                | Description                                                                                                                                                                          |
 | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `ovn_nb_addr` / `ovn_sb_addr` | OVN Northbound / Southbound DB endpoint(s). The NB and SB databases run clustered via OVSDB RAFT across the first three nodes (client ports 6641/6642, RAFT ports 6643/6644). Each node's config lists all three quorum endpoints so the loss of one DB node does not stall the control plane. A single `tcp:IP:6641` string remains valid for single-node dev. |
 | `external_interface` | Linux bridge that owns the WAN uplink on this node (e.g. `br-wan`, `br-public`). The physical NIC is enslaved to this bridge — not configured here. Different nodes may differ. |

 ## Pool Selection Logic
@@ -506,26 +603,10 @@ When an instance needs a public IP:

 ## IPAM Storage

-Each pool gets a NATS KV entry in bucket `spinifex-external-ipam`, keyed by pool
-name. Allocation uses CAS (Compare-And-Set) for lock-free concurrent access:
-
-```json
-{
-  "pool_name": "wan",
-  "range_start": "192.168.1.150",
-  "range_end": "192.168.1.250",
-  "allocated": {
-    "192.168.1.150": { "type": "gateway" },
-    "192.168.1.151": {
-      "type": "auto_assign",
-      "eni_id": "eni-abc",
-      "instance_id": "i-123"
-    }
-  }
-}
-```
-
-Pools are initialized from `spinifex.toml` on vpcd startup (idempotent).
+Pool allocation state is stored durably in the cluster (NATS KV bucket
+`spinifex-external-ipam`, one entry per pool) and survives restarts. Each
+allocation records the ENI and instance holding the address. Pools are
+initialized from `spinifex.toml` on vpcd startup (idempotent).

 ## Deployment Examples

@@ -1046,6 +1127,27 @@ sudo ovn-trace ext-vpc-{vpcId} \
   arp.op==1 && arp.spa==192.168.1.13 && arp.tpa==192.168.1.201'
 ```

+### OVN DB RAFT cluster (NB/SB replication)
+
+The NB and SB databases run clustered across the first three nodes via native
+OVSDB RAFT. A 3-node quorum tolerates the loss of one DB node; check status when
+the control plane stalls.
+
+```bash
+# NB cluster status: expect 3 servers and exactly one "Role: leader"
+sudo ovn-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
+
+# SB cluster status
+sudo ovn-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound
+
+# Drive a CLI at the whole quorum (fails over past a dead node)
+sudo ovn-nbctl --db=tcp:10.1.3.181:6641,tcp:10.1.3.182:6641,tcp:10.1.3.183:6641 show
+```
+
+A node showing only itself under `Servers:` never joined the cluster — confirm
+it was bootstrapped with `--db-cluster-remote-addr` pointing at the creator and
+that RAFT ports 6643/6644 are reachable between DB nodes.
+
 ### OVS (datapath / physical wiring)

 ```bash
@@ -1314,12 +1416,10 @@ OVNSB commit failed, force recompute next time.
 Repeated millions of times. Port binding never happens (`up: false`).

 **Cause:** Stale entries in the OVN Southbound DB (old chassis records, port
-bindings, datapath bindings) conflict with ovn-controller's expected state.
-Happens when `reset-dev-env.sh` cleans NB but not SB, or after ungraceful
-shutdowns.
+bindings, datapath bindings) conflict with ovn-controller's expected state,
+typically after an ungraceful shutdown.

-**Fix:** Delete both OVN DB files and restart. `reset-dev-env.sh` does this
-automatically:
+**Fix:** Delete both OVN DB files and restart:

 ```bash
 sudo systemctl stop ovn-central ovn-controller
@@ -1328,21 +1428,6 @@ sudo systemctl start ovn-central ovn-controller
 # vpcd reconcile will recreate the NB topology on next startup
 ```

-### Chassis Name Mismatch
-
-After mulga-999, vpcd uses the OVS-managed UUID (persisted at
-`/etc/openvswitch/system-id.conf` by the `openvswitch-switch` package and
-re-applied on every boot) as the canonical chassis identity. The chassis name
-across OVS `external_ids:system-id`, OVN SB `Chassis.name`, and NB
-`Gateway_Chassis.chassis_name` is always the same value, so the mismatch
-class that this section previously documented can no longer recur.
-
-`ReconcileFromKV` runs a `reconcileGatewayChassis` pre-step on every vpcd
-startup that deletes any stale `Gateway_Chassis` row left over from earlier
-installs (where `setup-ovn.sh` fabricated `chassis-$(hostname -s)`) and
-re-binds every gateway LRP against the live SBDB chassis. Pre-mulga-999
-brokenness recovers automatically on the first vpcd restart after upgrade —
-no manual `ovn-nbctl destroy` required.

 ### WAN NIC Not Enslaved to a Bridge

@@ -16,7 +16,7 @@ sections:
 resources:
  - title: "Spinifex Repository"
    url: "https://github.com/mulgadc/spinifex"
-  - title: "Terraform AWS Provider — ECR"
+  - title: "Terraform AWS Provider"
    url: "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecr_repository"
  - title: "OCI Distribution Spec"
    url: "https://github.com/opencontainers/distribution-spec"
@@ -62,11 +62,32 @@ A **task definition** describes one or more containers (image, CPU/memory, ports
 ECS v1 is deliberately minimal. Keep these in mind:

 - **No service discovery.** `serviceRegistries` / Cloud Map integration is not implemented — reach a service through its load balancer, not a DNS name.
- **`awslogs` log driver is discarded.** Container stdout/stderr is not shipped to CloudWatch Logs; the log configuration is accepted but dropped.
- **No rolling deployments.** The reconciler keeps the desired count running but ignores `deploymentConfiguration` (`minimumHealthyPercent` / `maximumPercent`); an `UpdateService` to a new revision replaces tasks without a health-gated rollout or circuit-breaker.
+- **Only the `json-file` log driver is collected.** Container stdout/stderr is captured host-side on the container instance (see [Logging](#logging)); it is not shipped to CloudWatch Logs. Any other driver (e.g. `awslogs`) is accepted for parity but its logs are discarded — `RegisterTaskDefinition` logs a warning naming the container so the drop is not silent.
 - **No capacity providers or managed scaling.** Capacity is the static total of your registered instances; there is no scale-out/in or ASG binding — you manage the instance count.
- **No distinct execution role.** ECR pulls use the instance role directly; `executionRoleArn` is not honoured separately.
- **`secrets[]` are silently dropped**, and tags set via `TagResource` are not persisted. Health-check settings beyond the target group default are not forwarded.
+- **`secrets[]` are rejected.** A task definition that declares container `secrets[]` fails `RegisterTaskDefinition` with `InvalidParameterException` rather than running without the secrets it expects. Tags set via `TagResource` are not persisted, and health-check settings beyond the target group default are not forwarded.
+
+### Logging
+
+Spinifex honours the host-side **`json-file`** log driver (the containerd default). A container's stdout/stderr is written on its container instance and retrievable there — no CloudWatch Logs path exists.
+
+To read a task's logs, find the container instance running it (`aws ecs describe-tasks` → `containerInstanceArn` → the EC2 instance), then on that host inspect the containerd task output. Containers are named `{taskId}-{containerName}` and carry `mulga.ecs.*` labels:
+
+```bash
+# On the container instance:
+ctr -n default containers ls                 # find {taskId}-{containerName}
+ctr -n default tasks ls
+journalctl -u containerd | grep <taskId>     # container stdout/stderr via the host journal
+```
+
+A task definition may still declare `awslogs` (or any other driver) for AWS compatibility — Spinifex accepts it, warns at registration that the driver is not implemented, and falls back to the host-side `json-file` behaviour.
+
+### Deployments
+
+An `UpdateService` to a new task-definition revision performs a **health-gated rolling update** honouring `deploymentConfiguration`: `minimumHealthyPercent` keeps that fraction of the desired count running while `maximumPercent` bounds how many extra tasks launch during the roll. Enabling the **deployment circuit breaker** fails a rollout whose tasks repeatedly fail to start, and — with `rollback` set — automatically reverts to the last-good task definition.
+
+### Execution role
+
+If a task definition sets `executionRoleArn`, the agent assumes that role to authorise ECR image pulls (instead of the container-instance role). When it is unset, pulls fall back to the instance role.

 ## Prerequisites

@@ -267,4 +288,4 @@ tofu apply -var 'gateway_url=https://<host-lan-ip>:9999'

 **Container cannot assume its task role.** Confirm the task definition sets `taskRoleArn` and the role is trusted by `ecs-tasks.amazonaws.com`. The agent injects `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`; a container that overrides this variable, or an SDK too old to honour it, will not pick up the credentials.

-**Expected a feature that is missing.** ECS v1 omits service discovery, `awslogs` shipping, rolling deployments, capacity providers, and a distinct execution role. See the Limitations in the Overview.
+**Expected a feature that is missing.** ECS v1 omits service discovery, CloudWatch Logs (`awslogs`) shipping, and capacity providers / managed scaling. See the Limitations in the Overview.
@@ -0,0 +1,222 @@
+---
+title: "IAM Groups"
+description: "Organise IAM users into groups and manage permissions once for the whole team."
+category: "Identity"
+tags:
+  - iam
+  - groups
+  - users
+  - policies
+  - security
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS IAM Groups Documentation"
+    url: "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_groups.html"
+---
+
+# IAM Groups
+
+> Organise IAM users into groups and manage permissions once for the whole team.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Groups](#groups)
+- [Membership](#membership)
+- [Group Policies](#group-policies)
+- [Putting It All Together](#putting-it-all-together)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+A group is a collection of IAM users. Policies attached to a group apply to every member, so instead of attaching the same policy to each developer individually, you attach it once to a `developers` group and manage membership.
+
+Members inherit both the group's **attached managed policies** and its **inline policies**, combined with any policies on the user itself. Policy evaluation is the same as everywhere else: an explicit `Deny` in any applicable policy wins, and anything not allowed is denied.
+
+Groups cannot be nested, and a group is not a principal — it cannot sign requests or own access keys. Like all IAM resources, groups are scoped to the account that creates them.
+
+## Prerequisites
+
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
+## Instructions
+
+## Groups
+
+### Create a Group
+
+```bash
+aws iam create-group --group-name developers
+```
+
+```json
+{
+  "Group": {
+    "Path": "/",
+    "GroupName": "developers",
+    "GroupId": "AGPA1A2B3C4D5E6F7890",
+    "Arn": "arn:aws:iam::000000000001:group/developers",
+    "CreateDate": "2026-07-03T10:00:00Z"
+  }
+}
+```
+
+Use `--path` to organise groups hierarchically:
+
+```bash
+aws iam create-group --group-name ops --path /teams/
+```
+
+### Get and List Groups
+
+`get-group` returns the group and its members:
+
+```bash
+aws iam get-group --group-name developers
+aws iam list-groups
+aws iam list-groups --path-prefix /teams/
+```
+
+### Delete a Group
+
+A group must have no members and no policies before it can be deleted — see [Cleaning up a group](#deleteconflict-when-deleting-a-group).
+
+```bash
+aws iam delete-group --group-name developers
+```
+
+## Membership
+
+Add and remove users:
+
+```bash
+aws iam add-user-to-group --group-name developers --user-name carol
+aws iam remove-user-from-group --group-name developers --user-name carol
+```
+
+List a group's members (the `Users` array of `get-group`):
+
+```bash
+aws iam get-group --group-name developers --query 'Users[].UserName'
+```
+
+List the groups a user belongs to:
+
+```bash
+aws iam list-groups-for-user --user-name carol --query 'Groups[].GroupName'
+```
+
+> **Group membership blocks user deletion.** `delete-user` returns `DeleteConflict` while the user is still in any group — remove them from all groups first.
+
+## Group Policies
+
+### Attach a Managed Policy
+
+```bash
+aws iam attach-group-policy --group-name developers \
+  --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
+
+aws iam list-attached-group-policies --group-name developers
+
+aws iam detach-group-policy --group-name developers \
+  --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
+```
+
+See [IAM Users and Policies](/docs/iam-users-and-policies) for creating managed policies and the policy document format.
+
+### Inline Group Policies
+
+Inline policies live inside the group and are deleted with it:
+
+```bash
+aws iam put-group-policy --group-name developers \
+  --policy-name keypair-mgmt \
+  --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["ec2:CreateKeyPair","ec2:DeleteKeyPair"],"Resource":"*"}]}'
+
+aws iam list-group-policies --group-name developers
+aws iam get-group-policy --group-name developers --policy-name keypair-mgmt
+aws iam delete-group-policy --group-name developers --policy-name keypair-mgmt
+```
+
+Members are authorised by the union of the group's attached and inline policies. A user in `developers` with the policies above can call `ec2:DescribeInstances` (via the attached `EC2ReadOnly`) and `ec2:CreateKeyPair` (via the inline policy), while anything not granted — say `ec2:RunInstances` — is denied.
+
+## Putting It All Together
+
+Onboard a team with shared permissions:
+
+```bash
+# 1. Create the group and grant it permissions
+aws iam create-group --group-name developers
+aws iam attach-group-policy --group-name developers \
+  --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
+
+# 2. Create users and add them to the group
+aws iam create-user --user-name carol
+aws iam add-user-to-group --group-name developers --user-name carol
+
+# 3. Create access keys and configure profiles as usual
+aws iam create-access-key --user-name carol
+
+# 4. Verify: carol can describe instances via group policy, nothing more
+AWS_PROFILE=spinifex-carol aws ec2 describe-instances
+AWS_PROFILE=spinifex-carol aws ec2 run-instances --image-id ami-123 --instance-type t3.micro
+# ^ AccessDenied — not granted by the group
+```
+
+To change the whole team's permissions later, edit the group's policies once — no per-user changes needed.
+
+## Troubleshooting
+
+### DeleteConflict When Deleting a Group
+
+The group still has members, attached policies, or inline policies. Empty it first:
+
+```bash
+# Members
+aws iam get-group --group-name developers --query 'Users[].UserName'
+aws iam remove-user-from-group --group-name developers --user-name carol
+
+# Attached policies
+aws iam list-attached-group-policies --group-name developers
+aws iam detach-group-policy --group-name developers --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
+
+# Inline policies
+aws iam list-group-policies --group-name developers
+aws iam delete-group-policy --group-name developers --policy-name keypair-mgmt
+
+aws iam delete-group --group-name developers
+```
+
+### DeleteConflict When Deleting a User
+
+Group membership counts as a subordinate entity, alongside access keys and attached policies. Find and leave the user's groups:
+
+```bash
+aws iam list-groups-for-user --user-name carol
+aws iam remove-user-from-group --group-name developers --user-name carol
+```
+
+### NoSuchEntity
+
+The group or user referenced does not exist — both `get-group` on a missing group and `add-user-to-group` with a missing user return this. Check spelling with `list-groups` / `list-users`.
+
+### Member Still Denied After Attaching a Group Policy
+
+Check the policy landed on the right group and the user is actually a member:
+
+```bash
+aws iam list-groups-for-user --user-name carol
+aws iam list-attached-group-policies --group-name developers
+aws iam list-group-policies --group-name developers
+```
+
+Remember an explicit `Deny` in any policy that applies to the user — their own or any of their groups' — overrides the `Allow`.
@@ -0,0 +1,376 @@
+---
+title: "IAM Roles and Instance Profiles"
+description: "Create IAM roles with trust policies, wrap them in instance profiles, and grant EC2 instances credentials without static keys."
+category: "Identity"
+tags:
+  - iam
+  - roles
+  - instance profiles
+  - trust policies
+  - credentials
+  - security
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS IAM Roles Documentation"
+    url: "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html"
+  - title: "AWS Instance Profiles Documentation"
+    url: "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html"
+---
+
+# IAM Roles and Instance Profiles
+
+> Create IAM roles with trust policies, wrap them in instance profiles, and grant EC2 instances credentials without static keys.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Roles](#roles)
+- [Granting Permissions to a Role](#granting-permissions-to-a-role)
+- [Instance Profiles](#instance-profiles)
+- [Launching an Instance with a Role](#launching-an-instance-with-a-role)
+- [Managing Profile Associations at Runtime](#managing-profile-associations-at-runtime)
+- [Assuming a Role](#assuming-a-role)
+- [Cleaning Up](#cleaning-up)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+A role is an IAM identity with permissions but no long-lived credentials. Instead of an access key pair, a role has a **trust policy** declaring who may assume it; whoever assumes the role receives short-lived, auto-rotating STS credentials.
+
+An **instance profile** is the container that binds a role to an EC2 instance. An instance launched with a profile gets the role's credentials delivered through [IMDS](/docs/imds) — no static keys baked into the image, no `~/.aws/credentials` inside the guest.
+
+The full arc is: create a role → attach permissions → wrap it in an instance profile → launch an instance with the profile → the guest picks up credentials automatically. Each instance profile holds exactly **one role** (matching AWS).
+
+Like all IAM resources, roles and instance profiles are scoped to the account that creates them.
+
+## Prerequisites
+
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
+## Instructions
+
+## Roles
+
+### Create a Role
+
+Every role needs a trust policy. For a role that EC2 instances will use, trust the `ec2.amazonaws.com` service principal:
+
+```bash
+cat > /tmp/ec2-trust.json << 'EOF'
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com" },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+EOF
+
+aws iam create-role --role-name app-server \
+  --assume-role-policy-document file:///tmp/ec2-trust.json \
+  --description "Role for app servers"
+```
+
+```json
+{
+  "Role": {
+    "Path": "/",
+    "RoleName": "app-server",
+    "RoleId": "AROA1A2B3C4D5E6F7890",
+    "Arn": "arn:aws:iam::000000000001:role/app-server",
+    "CreateDate": "2026-07-03T10:00:00Z",
+    "AssumeRolePolicyDocument": {
+      "Version": "2012-10-17",
+      "Statement": [
+        {
+          "Effect": "Allow",
+          "Principal": { "Service": "ec2.amazonaws.com" },
+          "Action": "sts:AssumeRole"
+        }
+      ]
+    },
+    "Description": "Role for app servers",
+    "MaxSessionDuration": 3600
+  }
+}
+```
+
+To let a user (rather than EC2) assume the role, trust an `AWS` principal instead:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "AWS": "arn:aws:iam::000000000001:user/admin" },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+```
+
+**Trust policy validation:** Spinifex rejects `NotPrincipal`, `NotAction`, empty `Principal` blocks, and empty-string `Action` elements at write time with `MalformedPolicyDocument`. `Condition` blocks are rejected except the `StringEquals` form used for web identity federation.
+
+### Get, List, and Update Roles
+
+```bash
+aws iam get-role --role-name app-server
+aws iam list-roles
+aws iam list-roles --path-prefix /services/
+```
+
+Update the description or the maximum STS session duration (3600 seconds by default):
+
+```bash
+aws iam update-role --role-name app-server \
+  --description "Updated" --max-session-duration 7200
+```
+
+Replace the trust policy on an existing role:
+
+```bash
+aws iam update-assume-role-policy --role-name app-server \
+  --policy-document file:///tmp/ec2-trust.json
+```
+
+## Granting Permissions to a Role
+
+A freshly created role can do nothing. Grant permissions the same two ways as users: attach managed policies, or embed inline policies.
+
+### Attach a Managed Policy
+
+```bash
+aws iam attach-role-policy --role-name app-server \
+  --policy-arn arn:aws:iam::000000000001:policy/S3ReadOnly
+
+aws iam list-attached-role-policies --role-name app-server
+aws iam detach-role-policy --role-name app-server \
+  --policy-arn arn:aws:iam::000000000001:policy/S3ReadOnly
+```
+
+See [IAM Users and Policies](/docs/iam-users-and-policies) for creating managed policies and the policy document format.
+
+### Inline Role Policies
+
+Inline policies live inside the role and are deleted with it:
+
+```bash
+aws iam put-role-policy --role-name app-server \
+  --policy-name ec2-describe \
+  --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["ec2:DescribeInstances"],"Resource":"*"}]}'
+
+aws iam list-role-policies --role-name app-server
+aws iam get-role-policy --role-name app-server --policy-name ec2-describe
+aws iam delete-role-policy --role-name app-server --policy-name ec2-describe
+```
+
+## Instance Profiles
+
+Create a profile and add the role to it:
+
+```bash
+aws iam create-instance-profile --instance-profile-name app-server-profile
+aws iam add-role-to-instance-profile \
+  --instance-profile-name app-server-profile --role-name app-server
+```
+
+Inspect it — the `Roles` array shows the bound role:
+
+```bash
+aws iam get-instance-profile --instance-profile-name app-server-profile
+aws iam list-instance-profiles
+aws iam list-instance-profiles-for-role --role-name app-server
+```
+
+A profile holds at most one role; adding a second returns `LimitExceeded`. To swap roles, remove the current one first:
+
+```bash
+aws iam remove-role-from-instance-profile \
+  --instance-profile-name app-server-profile --role-name app-server
+```
+
+## Launching an Instance with a Role
+
+Pass the profile by name (or ARN) at launch:
+
+```bash
+aws ec2 run-instances \
+  --image-id ami-0dd52c90440ff4150 \
+  --instance-type t3.micro \
+  --subnet-id subnet-a0e5fc381376d82a1 \
+  --iam-instance-profile Name=app-server-profile
+```
+
+The instance description shows the association:
+
+```bash
+aws ec2 describe-instances --instance-ids i-d2e09ff7de71b6341 \
+  --query 'Reservations[0].Instances[0].IamInstanceProfile'
+```
+
+```json
+{
+  "Arn": "arn:aws:iam::000000000001:instance-profile/app-server-profile",
+  "Id": "AIPA1A2B3C4D5E6F7890"
+}
+```
+
+Inside the guest, the AWS CLI and SDKs pick up the role's credentials from IMDS with no configuration — see [IMDS](/docs/imds) for fetching them manually, rotation timing, and limits.
+
+## Managing Profile Associations at Runtime
+
+Profiles can be attached to, removed from, or swapped on a **running** instance — no relaunch needed.
+
+List associations:
+
+```bash
+aws ec2 describe-iam-instance-profile-associations \
+  --filters Name=instance-id,Values=i-d2e09ff7de71b6341
+```
+
+```json
+{
+  "IamInstanceProfileAssociations": [
+    {
+      "AssociationId": "iip-assoc-9d7a617c605e337cd",
+      "InstanceId": "i-d2e09ff7de71b6341",
+      "IamInstanceProfile": {
+        "Arn": "arn:aws:iam::000000000001:instance-profile/app-server-profile"
+      },
+      "State": "associated"
+    }
+  ]
+}
+```
+
+Attach a profile to an instance launched without one:
+
+```bash
+aws ec2 associate-iam-instance-profile \
+  --instance-id i-d2e09ff7de71b6341 \
+  --iam-instance-profile Name=app-server-profile
+```
+
+Swap to a different profile (returns a new association ID):
+
+```bash
+aws ec2 replace-iam-instance-profile-association \
+  --association-id iip-assoc-9d7a617c605e337cd \
+  --iam-instance-profile Name=other-profile
+```
+
+Remove the profile — the guest's `iam/` metadata subtree starts returning 404:
+
+```bash
+aws ec2 disassociate-iam-instance-profile \
+  --association-id iip-assoc-9d7a617c605e337cd
+```
+
+## Assuming a Role
+
+Users and services can assume a role directly with STS, provided the trust policy allows their principal:
+
+```bash
+aws sts assume-role \
+  --role-arn arn:aws:iam::000000000001:role/deploy \
+  --role-session-name release-42
+```
+
+```json
+{
+  "Credentials": {
+    "AccessKeyId": "ASIA1A2B3C4D5E6F7890",
+    "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+    "SessionToken": "IQoJb3JpZ2luX2VjE...",
+    "Expiration": "2026-07-03T11:00:00Z"
+  },
+  "AssumedRoleUser": {
+    "AssumedRoleId": "AROA1A2B3C4D5E6F7890:release-42",
+    "Arn": "arn:aws:sts::000000000001:assumed-role/deploy/release-42"
+  }
+}
+```
+
+The returned `ASIA`-prefixed credentials are temporary, valid up to the role's `MaxSessionDuration`. See [STS and Temporary Credentials](/docs/sts) for session durations, using the credentials, and the other STS flows.
+
+## Cleaning Up
+
+Deletion order matters — a role must be empty and unreferenced before it can be deleted:
+
+```bash
+# 1. Remove the role from any instance profiles
+aws iam remove-role-from-instance-profile \
+  --instance-profile-name app-server-profile --role-name app-server
+aws iam delete-instance-profile --instance-profile-name app-server-profile
+
+# 2. Delete inline policies and detach managed policies
+aws iam delete-role-policy --role-name app-server --policy-name ec2-describe
+aws iam detach-role-policy --role-name app-server \
+  --policy-arn arn:aws:iam::000000000001:policy/S3ReadOnly
+
+# 3. Delete the role
+aws iam delete-role --role-name app-server
+```
+
+## Troubleshooting
+
+### MalformedPolicyDocument When Creating a Role
+
+The trust policy is invalid. In addition to the JSON checks that apply to all policies, trust policies must not use `NotPrincipal`, `NotAction`, an empty `Principal` block, or an empty-string `Action`. `Condition` blocks are rejected except `StringEquals` conditions used with web identity federation.
+
+### DeleteConflict When Deleting a Role
+
+The role still has attached managed policies, inline policies, or is bound to an instance profile. Remove all three first:
+
+```bash
+aws iam list-attached-role-policies --role-name app-server
+aws iam list-role-policies --role-name app-server
+aws iam list-instance-profiles-for-role --role-name app-server
+```
+
+### DeleteConflict When Deleting an Instance Profile
+
+The profile still contains a role:
+
+```bash
+aws iam remove-role-from-instance-profile \
+  --instance-profile-name app-server-profile --role-name app-server
+aws iam delete-instance-profile --instance-profile-name app-server-profile
+```
+
+### LimitExceeded When Adding a Role to a Profile
+
+The profile already holds a role — the limit is one per profile. Remove the existing role first, or create a second profile.
+
+### InvalidIamInstanceProfile.NotFound at Launch
+
+The profile name or ARN passed to `run-instances` does not exist in your account. Check the spelling:
+
+```bash
+aws iam list-instance-profiles
+```
+
+### Instance Has No Credentials in IMDS
+
+If the `iam/` metadata subtree returns 404, the instance has no associated profile. Attach one without relaunching:
+
+```bash
+aws ec2 associate-iam-instance-profile \
+  --instance-id i-d2e09ff7de71b6341 \
+  --iam-instance-profile Name=app-server-profile
+```
+
+If the profile is associated but contains no role, the `security-credentials/` listing is empty; add the role and credentials appear on the next request.
@@ -1,7 +1,7 @@
 ---
 title: "IAM Users and Policies"
 description: "Create IAM users, manage access keys, and control permissions with policies."
-category: "Administration"
+category: "Identity"
 tags:
  - iam
  - users
@@ -30,31 +30,29 @@ resources:
 - [Policies](#policies)
 - [Attaching Policies](#attaching-policies)
 - [Putting It All Together](#putting-it-all-together)
- [Command Reference](#command-reference)
 - [Troubleshooting](#troubleshooting)

 ---

 ## Overview

-Spinifex implements AWS-compatible IAM with 16 operations covering user management, access key lifecycle, policy CRUD, and policy attachment. All IAM resources are scoped to the account that creates them — users in one account cannot see or modify resources in another.
+Spinifex implements AWS-compatible IAM covering user management, access key lifecycle, policy CRUD, and policy attachment. Inline user policies (`put-user-policy`), user and policy tagging, [groups](/docs/iam-groups), and [roles and instance profiles](/docs/iam-roles-and-instance-profiles) are also supported. All IAM resources are scoped to the account that creates them — users in one account cannot see or modify resources in another.

 When you create an account with `spx admin account create`, Spinifex bootstraps a root user with an `AdministratorAccess` policy and writes the credentials to `~/.aws/credentials`. From there, you use the standard AWS CLI to manage additional users and permissions.

 **How authentication works:** Every AWS CLI request is signed with SigV4 using an access key pair. The gateway verifies the signature, resolves the caller's account, and evaluates attached policies before routing the request. The root user (account `000000000000`) bypasses policy evaluation entirely.

-## Instructions
-
 ## Prerequisites

- A running Spinifex cluster with the AWS gateway started
- An account created via `spx admin account create` (see [Creating Accounts](/docs/creating-accounts))
- AWS CLI configured with the account's profile:
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:

 ```bash
-export AWS_PROFILE=spinifex-myteam
+export AWS_PROFILE=spinifex
 ```

+## Instructions
+
 ## Users

 ### Create a User
@@ -143,7 +141,17 @@ aws iam create-access-key --user-name alice

 > **The secret access key is only shown once.** Save it immediately. If lost, delete the key and create a new one.

-Add the credentials to `~/.aws/credentials` under a new profile and set `AWS_PROFILE` to use them.
+Configure a profile for the new key. A Spinifex profile also needs the gateway endpoint and CA bundle (copy the values from your existing `spinifex-*` profile in `~/.aws/config`):
+
+```bash
+aws configure set aws_access_key_id AKIA1A2B3C4D5E6F7890ABCD --profile spinifex-alice
+aws configure set aws_secret_access_key wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY --profile spinifex-alice
+aws configure set region ap-southeast-2 --profile spinifex-alice
+aws configure set endpoint_url https://localhost:9999 --profile spinifex-alice
+aws configure set ca_bundle /var/lib/spinifex/config/ca.pem --profile spinifex-alice
+```
+
+Then set `AWS_PROFILE=spinifex-alice` to use it.

 ### List Access Keys

@@ -159,7 +167,9 @@ Create a second key, update your configuration, then delete the old one:
 # 1. Create new key (while old key still works)
 aws iam create-access-key --user-name alice

-# 2. Update ~/.aws/credentials with the new key
+# 2. Update the profile with the new key
+aws configure set aws_access_key_id AKIA_NEW_KEY_ID --profile spinifex-alice
+aws configure set aws_secret_access_key NEW_SECRET --profile spinifex-alice

 # 3. Verify the new key works
 AWS_PROFILE=spinifex-alice aws sts get-caller-identity
@@ -433,9 +443,13 @@ aws iam create-policy \
 aws iam attach-user-policy --user-name dev-carol \
  --policy-arn arn:aws:iam::000000000001:policy/DeveloperAccess

-# 5. Configure the developer's AWS CLI
-aws configure --profile spinifex-carol
-# Enter the access key and secret from step 2
+# 5. Configure the developer's AWS CLI profile with the key from step 2.
+#    endpoint_url and ca_bundle: copy from an existing spinifex-* profile in ~/.aws/config
+aws configure set aws_access_key_id AKIA_CAROL_KEY_ID --profile spinifex-carol
+aws configure set aws_secret_access_key CAROL_SECRET --profile spinifex-carol
+aws configure set region ap-southeast-2 --profile spinifex-carol
+aws configure set endpoint_url https://localhost:9999 --profile spinifex-carol
+aws configure set ca_bundle /var/lib/spinifex/config/ca.pem --profile spinifex-carol

 # 6. Verify
 AWS_PROFILE=spinifex-carol aws ec2 describe-instances
@@ -443,35 +457,14 @@ AWS_PROFILE=spinifex-carol aws ec2 terminate-instances --instance-ids i-123
 # ^ This will be denied by the NoTerminate statement
 ```

-## Command Reference
-
-| Command                       | Key Parameters                               | Notes                             |
-| ----------------------------- | -------------------------------------------- | --------------------------------- |
-| `create-user`                 | `--user-name`, `--path`                      | Path default: `/`                 |
-| `get-user`                    | `--user-name`                                |                                   |
-| `list-users`                  | `--path-prefix`                              |                                   |
-| `delete-user`                 | `--user-name`                                | Must have no keys or policies     |
-| `create-access-key`           | `--user-name`                                | Max 2 per user; secret shown once |
-| `list-access-keys`            | `--user-name`                                |                                   |
-| `delete-access-key`           | `--user-name`, `--access-key-id`             |                                   |
-| `update-access-key`           | `--user-name`, `--access-key-id`, `--status` | `Active` or `Inactive`            |
-| `create-policy`               | `--policy-name`, `--policy-document`         | Max 6144 bytes                    |
-| `get-policy`                  | `--policy-arn`                               | Metadata only                     |
-| `get-policy-version`          | `--policy-arn`, `--version-id`               | Use `v1`; includes document       |
-| `list-policies`               |                                              |                                   |
-| `delete-policy`               | `--policy-arn`                               | Must not be attached              |
-| `attach-user-policy`          | `--user-name`, `--policy-arn`                | Idempotent                        |
-| `detach-user-policy`          | `--user-name`, `--policy-arn`                |                                   |
-| `list-attached-user-policies` | `--user-name`                                |                                   |
-
 ## Troubleshooting

 ### AccessDenied on IAM Commands

-The calling user must have IAM permissions. Attach a policy with `iam:*` actions, or use the root account profile:
+The calling user must have IAM permissions. Attach a policy with `iam:*` actions, or use the account's admin profile:

 ```bash
-export AWS_PROFILE=spinifex-myteam
+export AWS_PROFILE=spinifex
 aws iam list-users
 ```

@@ -494,7 +487,7 @@ If the key was deleted, create a new one.

 ### DeleteConflict When Deleting a User

-The user still has access keys or attached policies. Remove them first:
+The user still has access keys, attached policies, or [group memberships](/docs/iam-groups). Remove them first:

 ```bash
 # Check for access keys
@@ -502,6 +495,9 @@ aws iam list-access-keys --user-name alice

 # Check for attached policies
 aws iam list-attached-user-policies --user-name alice
+
+# Check for group memberships
+aws iam list-groups-for-user --user-name alice
 ```

 ### DeleteConflict When Deleting a Policy
@@ -0,0 +1,349 @@
+---
+title: "IMDS (Instance Metadata Service)"
+description: "Access instance metadata, user data, and IAM role credentials from inside a guest VM with IMDSv2."
+category: "Identity"
+tags:
+  - imds
+  - metadata
+  - instance identity
+  - iam roles
+  - credentials
+  - security
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS IMDS Documentation"
+    url: "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html"
+  - title: "AWS IMDSv2 Deep Dive"
+    url: "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html"
+---
+
+# Instance Metadata Service (IMDS)
+
+> Access instance metadata, user data, and IAM role credentials from inside a guest VM with IMDSv2.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Using IMDS from a Guest](#using-imds-from-a-guest)
+- [Metadata Paths](#metadata-paths)
+- [User Data](#user-data)
+- [IAM Role Credentials](#iam-role-credentials)
+- [Instance Identity Document](#instance-identity-document)
+- [Metadata Options](#metadata-options)
+- [How It Works](#how-it-works)
+- [Limits and Defaults](#limits-and-defaults)
+- [Command Reference](#command-reference)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Every running guest VM can reach the Instance Metadata Service at `http://169.254.169.254`, exactly as on EC2. There is no in-VM agent to install and no in-guest route configuration — DHCP and fully static guests reach it identically.
+
+IMDS answers questions a workload asks about itself: instance ID, instance type, private and public IPs, hostname, availability zone, security groups, the launch SSH key, user data, and — when the instance has an IAM instance profile — short-lived, auto-rotating role credentials.
+
+**Spinifex is IMDSv2-only.** Every read requires a session token obtained with a `PUT` request. A tokenless (IMDSv1-style) `GET` returns `401 Unauthorized` with an empty body.
+
+Requests are attributed to an instance by the network interface they arrive on and tokens are bound to that interface, so one instance can never read another's metadata or replay its tokens.
+
+## Prerequisites
+
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- A running instance to query from
+- For IAM role credentials: an instance launched with `--iam-instance-profile`
+
+## Instructions
+
+## Using IMDS from a Guest
+
+All commands in this section run **inside the guest VM**.
+
+### Get a Session Token
+
+Request a token with a TTL between 1 and 21600 seconds (6 hours):
+
+```bash
+TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" \
+  -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+```
+
+The token endpoint only accepts `PUT` — a `GET` returns `405 Method Not Allowed`. A missing or out-of-range TTL header returns `400 Bad Request`.
+
+### Read Metadata
+
+Send the token back in the `X-aws-ec2-metadata-token` header on every read:
+
+```bash
+curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
+  http://169.254.169.254/latest/meta-data/instance-id
+```
+
+```
+i-0a1b2c3d4e5f67890
+```
+
+Directory paths (ending in `/`) return a newline-separated listing of children:
+
+```bash
+curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
+  http://169.254.169.254/latest/meta-data/
+```
+
+### One-Liner Pattern
+
+For scripts, mint a short-lived token inline:
+
+```bash
+imds() {
+  local token=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" \
+    -H "X-aws-ec2-metadata-token-ttl-seconds: 60")
+  curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/$1"
+}
+
+imds meta-data/instance-id
+imds meta-data/local-ipv4
+imds meta-data/placement/availability-zone
+```
+
+## Metadata Paths
+
+`GET /` lists supported API versions (`2021-07-15`, `latest`). Any dated version segment (e.g. `/2016-09-02/...`, as probed by cloud-init) aliases to `/latest`.
+
+Commonly used paths under `/latest/meta-data/`:
+
+| Path | Returns |
+| ---- | ------- |
+| `instance-id` | Instance ID |
+| `instance-type` | Instance type (e.g. `t3.micro`) |
+| `ami-id` | Image the instance was launched from |
+| `ami-launch-index` | Launch index within the reservation (`0..n-1`) |
+| `reservation-id` | Reservation ID |
+| `instance-life-cycle` | `spot` or `on-demand` |
+| `local-ipv4` | Primary private IP |
+| `public-ipv4` | Elastic/public IP; 404 when none |
+| `public-hostname` | Mirrors `public-ipv4`; 404 when no public IP |
+| `mac` | Primary interface MAC address |
+| `hostname`, `local-hostname` | `ip-<dashed-ip>.<region>.compute.internal` |
+| `security-groups` | Security group names, one per line |
+| `placement/availability-zone` | Availability zone |
+| `placement/region` | Region (AZ with trailing letter stripped) |
+| `services/domain`, `services/partition` | `amazonaws.com` / `aws` |
+| `public-keys/0/openssh-key` | Launch key pair's SSH public key; 404 if no key pair or the key was deleted |
+| `iam/info` | Instance profile ARN and ID; 404 if no profile |
+| `iam/security-credentials/<role>` | Temporary role credentials (see below) |
+| `network/interfaces/macs/<mac>/...` | Primary interface subtree: `interface-id`, `owner-id`, `subnet-id`, `vpc-id`, `local-ipv4s`, `security-group-ids`, `subnet-ipv4-cidr-block`, `vpc-ipv4-cidr-block`, and more |
+
+The `network/interfaces/macs/` subtree covers the **primary interface only**; querying another MAC returns 404 (multi-ENI metadata is deferred).
+
+Paths that intentionally return **404**: `tags/instance/*`, `block-device-mapping/*`, `placement/{group-name,partition-number,availability-zone-id,host-id}`, `instance-action`, and `spot/{instance-action,termination-time}` (404 is the correct "no interruption scheduled" answer for spot pollers).
+
+## User Data
+
+User data supplied at launch (`run-instances --user-data`) is served at:
+
+```bash
+curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
+  http://169.254.169.254/latest/user-data
+```
+
+Returns the decoded user data, or 404 if the instance was launched without any. cloud-init consumes this automatically at first boot.
+
+## IAM Role Credentials
+
+Instances launched with an IAM instance profile get short-lived, auto-rotating credentials through IMDS — no static keys baked into the image. Creating roles and instance profiles is covered in [IAM Roles and Instance Profiles](/docs/iam-roles-and-instance-profiles).
+
+```bash
+# Discover the role name
+ROLE=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
+  http://169.254.169.254/latest/meta-data/iam/security-credentials/)
+
+# Fetch credentials
+curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
+  "http://169.254.169.254/latest/meta-data/iam/security-credentials/$ROLE"
+```
+
+```json
+{
+  "Code": "Success",
+  "LastUpdated": "2026-07-03T10:00:00Z",
+  "Type": "AWS-HMAC",
+  "AccessKeyId": "ASIA1A2B3C4D5E6F7890",
+  "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+  "Token": "IQoJb3JpZ2luX2VjE...",
+  "Expiration": "2026-07-03T11:00:00Z",
+  "AccountId": "000000000001"
+}
+```
+
+Credentials are ASIA-prefixed temporary STS credentials valid for **1 hour**, re-minted automatically **5 minutes before expiry**. The AWS SDKs and CLI pick them up with no configuration:
+
+```bash
+# Inside the guest, with no ~/.aws/credentials:
+aws sts get-caller-identity
+```
+
+If the instance has no profile, the whole `iam/` subtree returns 404 (and is omitted from the `meta-data/` listing).
+
+## Instance Identity Document
+
+An unsigned identity document (schema `2017-09-30`) is available at:
+
+```bash
+curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
+  http://169.254.169.254/latest/dynamic/instance-identity/document
+```
+
+```json
+{
+  "accountId": "000000000001",
+  "architecture": "x86_64",
+  "availabilityZone": "ap-southeast-2a",
+  "imageId": "ami-0123456789abcdef0",
+  "instanceId": "i-0a1b2c3d4e5f67890",
+  "instanceType": "t3.micro",
+  "pendingTime": "2026-07-03T09:58:12Z",
+  "privateIp": "10.0.1.15",
+  "region": "ap-southeast-2",
+  "version": "2017-09-30"
+}
+```
+
+The signed forms (`signature`, `pkcs7`, `rsa2048`) currently return 404 — they require a per-cluster signing key and land with EKS IRSA support.
+
+## Metadata Options
+
+Metadata options are managed from the **control plane** with the standard EC2 commands. Because the platform is IMDSv2-only, the only mutable knob is the PUT response hop limit.
+
+### Inspect
+
+```bash
+aws ec2 describe-instances --instance-ids i-0a1b2c3d4e5f67890 \
+  --query 'Reservations[0].Instances[0].MetadataOptions'
+```
+
+```json
+{
+  "State": "applied",
+  "HttpTokens": "required",
+  "HttpPutResponseHopLimit": 1,
+  "HttpEndpoint": "enabled",
+  "HttpProtocolIpv6": "disabled",
+  "InstanceMetadataTags": "disabled"
+}
+```
+
+### Set at Launch
+
+```bash
+aws ec2 run-instances \
+  --image-id ami-0123456789abcdef0 \
+  --instance-type t3.micro \
+  --metadata-options "HttpPutResponseHopLimit=2"
+```
+
+### Modify a Running or Stopped Instance
+
+```bash
+aws ec2 modify-instance-metadata-options \
+  --instance-id i-0a1b2c3d4e5f67890 \
+  --http-put-response-hop-limit 2
+```
+
+Raise the hop limit above the default of 1 when containers on the instance need IMDS access through an extra routing hop (e.g. containers on a bridge network fetching role credentials). Valid range is 1–64.
+
+Rejected settings:
+
+- `--http-tokens optional` → `UnsupportedOperation` (IMDSv2 cannot be relaxed)
+- `--http-endpoint disabled` → `UnsupportedOperation` (the endpoint cannot be turned off)
+- `--http-protocol-ipv6 enabled` → `UnsupportedOperation`
+- `--instance-metadata-tags enabled` → `UnsupportedOperation`
+- Hop limit outside 1–64 → `InvalidParameterValue`
+
+## How It Works
+
+Useful background for host-side troubleshooting:
+
+- The IMDS HTTP server runs inside the `spinifex-vpcd` service on each host, listening on `169.254.169.254:80` with one listener per instance's primary interface — attribution is structural, with no source-IP trust.
+- A freshly launched guest's metadata service comes up within a few seconds of launch; cloud-init's built-in retries absorb this window.
+- Session tokens and cached role credentials are not persisted. A vpcd restart drops them; SDKs and cloud-init transparently reissue. Datapath state on the host survives service restarts.
+
+## Limits and Defaults
+
+| Setting | Value |
+| ------- | ----- |
+| Token TTL | 1–21600 seconds (request-scoped, via header) |
+| Token binding | Issuing network interface; in-memory, not persisted |
+| Role credential lifetime | 3600 seconds, refreshed 5 minutes before expiry |
+| PUT response hop limit | Default 1, valid 1–64 |
+| `HttpTokens` | Always `required` (immutable) |
+| `HttpEndpoint` | Always `enabled` (immutable) |
+| Tokenless GET | `401 Unauthorized`, empty body |
+| `X-Forwarded-For` present | `403 Forbidden` |
+| Wrong method | `405 Method Not Allowed` |
+| Unknown/unsupported path | `404 Not Found` |
+
+## Troubleshooting
+
+### 401 Unauthorized on Every Read
+
+The request is missing a valid token. Common causes:
+
+- No `X-aws-ec2-metadata-token` header — IMDSv1-style access is not supported; obtain a token first.
+- The token expired — reissue with a fresh `PUT /latest/api/token`.
+- The token was issued to a different instance/interface — tokens are interface-bound and rejected elsewhere, identically to unknown tokens.
+- The token endpoint itself never 401s; if the `PUT` fails, check for a `400` (bad TTL header) instead.
+
+### 403 Forbidden
+
+The request carried an `X-Forwarded-For` header. IMDS rejects proxied requests outright; call it directly from the workload, not through a forward proxy.
+
+### Software in a Container Cannot Reach IMDS
+
+The default hop limit of 1 means the PUT response dies at the first routed hop, e.g. a Docker bridge network. Raise it from the control plane:
+
+```bash
+aws ec2 modify-instance-metadata-options \
+  --instance-id i-0a1b2c3d4e5f67890 \
+  --http-put-response-hop-limit 2
+```
+
+### UnsupportedOperation When Setting Metadata Options
+
+You attempted to relax IMDSv2 enforcement (`--http-tokens optional`) or disable the endpoint (`--http-endpoint disabled`). Neither is supported — the platform posture is fixed. Only the hop limit can be changed.
+
+### IAM Credentials Return 404, an Empty List, or "Code": "Failed"
+
+A 404 or empty list means the instance has no IAM instance profile, or the profile has no role. Verify from the control plane:
+
+```bash
+aws ec2 describe-instances --instance-ids i-0a1b2c3d4e5f67890 \
+  --query 'Reservations[0].Instances[0].IamInstanceProfile'
+```
+
+A credential body with `"Code": "Failed"` means the backend could not mint credentials (for example, the role was deleted after launch). Check the role still exists and the vpcd logs on the host for `IMDS: AssumeRoleForInstance failed`. See the IAM Roles and Instance Profiles guide for setting up profiles.
+
+### Connection Refused / Timeout to 169.254.169.254
+
+Metadata is served per-interface by `spinifex-vpcd` on the host. In order:
+
+1. Immediately after launch, wait a few seconds — the listener converges within one 15-second reconcile tick, and cloud-init retries through it.
+2. On the host, confirm vpcd is running and serving:
+
+```bash
+systemctl status spinifex-vpcd
+journalctl -u spinifex-vpcd | grep 'IMDS:'
+```
+
+Look for `IMDS: tap responder serving` (listener bound) and `IMDS: issued IMDSv2 token` (proof guest packets are traversing the datapath). A bound responder with no token issuance points at the host datapath rather than the HTTP service:
+
+```bash
+ovs-vsctl list-ports br-imds          # per-instance endpoint + patch ports
+ovs-vsctl list-ports br-int | grep imi-
+```
+
+### cloud-init Did Not Apply SSH Key or User Data
+
+cloud-init sources both from IMDS at first boot. Check `public-keys/0/openssh-key` and `user-data` respond from inside the guest (see [Using IMDS from a Guest](#using-imds-from-a-guest)), and review `/var/log/cloud-init.log` in the guest. A 404 on `public-keys/` means the instance was launched without `--key-name` or the key pair was since deleted.
@@ -0,0 +1,346 @@
+---
+title: "STS and Temporary Credentials"
+description: "Mint short-lived credentials with STS — assume roles, issue session tokens, and federate Kubernetes workloads with OIDC."
+category: "Identity"
+tags:
+  - sts
+  - iam
+  - temporary credentials
+  - assume role
+  - oidc
+  - security
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS STS Documentation"
+    url: "https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html"
+  - title: "AWS Temporary Security Credentials"
+    url: "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html"
+---
+
+# STS and Temporary Credentials
+
+> Mint short-lived credentials with STS — assume roles, issue session tokens, and federate Kubernetes workloads with OIDC.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Checking Your Identity](#checking-your-identity)
+- [Assuming a Role](#assuming-a-role)
+- [Using Temporary Credentials](#using-temporary-credentials)
+- [Session Tokens for Your Own User](#session-tokens-for-your-own-user)
+- [Web Identity Federation (IRSA)](#web-identity-federation-irsa)
+- [Trust Policy Validation](#trust-policy-validation)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+STS (Security Token Service) issues **temporary credentials**: an `ASIA`-prefixed access key, a secret key, and a session token, all of which expire together at a fixed time. Nothing to rotate, nothing to revoke — the credentials simply stop working.
+
+Spinifex supports the three main ways to obtain them:
+
+- **`assume-role`** — exchange your IAM user credentials for a [role's](/docs/iam-roles-and-instance-profiles) permissions, gated by the role's trust policy.
+- **`get-session-token`** — get a time-boxed copy of your own user's permissions.
+- **`assume-role-with-web-identity`** — exchange an OIDC ID token (a Kubernetes ServiceAccount token) for role credentials, with no IAM credentials at all.
+
+EC2 instances get role credentials a fourth way, automatically through [IMDS](/docs/imds) — no STS call needed in the guest.
+
+## Prerequisites
+
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
+## Instructions
+
+## Checking Your Identity
+
+`get-caller-identity` returns whoever signed the request — it needs no permissions and never fails for authorisation reasons, which makes it the standard "which credentials am I using?" check:
+
+```bash
+aws sts get-caller-identity
+```
+
+```json
+{
+    "UserId": "AIDA1A2B3C4D5E6F7890",
+    "Account": "000000000001",
+    "Arn": "arn:aws:iam::000000000001:user/admin"
+}
+```
+
+Run with temporary role credentials, the ARN switches to the assumed-role form — see [Using Temporary Credentials](#using-temporary-credentials).
+
+## Assuming a Role
+
+`assume-role` works when the target role's trust policy allows your principal. Create a role that trusts an IAM user (see [IAM Roles and Instance Profiles](/docs/iam-roles-and-instance-profiles) for the full role lifecycle):
+
+```bash
+cat > /tmp/deploy-trust.json << 'EOF'
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "AWS": "arn:aws:iam::000000000001:user/admin" },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+EOF
+
+aws iam create-role --role-name deploy \
+  --assume-role-policy-document file:///tmp/deploy-trust.json
+
+aws iam put-role-policy --role-name deploy --policy-name ec2-read \
+  --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["ec2:Describe*"],"Resource":"*"}]}'
+```
+
+Then assume it:
+
+```bash
+aws sts assume-role \
+  --role-arn arn:aws:iam::000000000001:role/deploy \
+  --role-session-name release-42
+```
+
+```json
+{
+    "Credentials": {
+        "AccessKeyId": "ASIA1A2B3C4D5E6F7890",
+        "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+        "SessionToken": "IQoJb3JpZ2luX2VjE...",
+        "Expiration": "2026-07-03T11:00:00Z"
+    },
+    "AssumedRoleUser": {
+        "AssumedRoleId": "AROA1A2B3C4D5E6F7890:release-42",
+        "Arn": "arn:aws:sts::000000000001:assumed-role/deploy/release-42"
+    },
+    "PackedPolicySize": 0
+}
+```
+
+The session name tags every session so audit logs can tell *who* used the role; it appears in the assumed-role ARN and the `UserId`.
+
+If the trust policy does not allow your principal, `assume-role` fails with `AccessDenied` — no matter what IAM permissions you hold.
+
+### Session Duration
+
+`--duration-seconds` must be between **900** (15 minutes) and the smaller of the role's `MaxSessionDuration` and **43200** (12 hours); the default is **3600** (1 hour). A value outside that window is rejected with `ValidationError`, so to run longer sessions, raise the role's ceiling first:
+
+```bash
+# 7200 > MaxSessionDuration (3600) → ValidationError
+aws sts assume-role --role-arn arn:aws:iam::000000000001:role/deploy \
+  --role-session-name long-build --duration-seconds 7200
+
+aws iam update-role --role-name deploy --max-session-duration 43200
+
+aws sts assume-role --role-arn arn:aws:iam::000000000001:role/deploy \
+  --role-session-name long-build --duration-seconds 7200
+```
+
+### Flags That Differ from AWS
+
+- `--external-id` and `--source-identity` are accepted and logged but **not enforced** — trust policies cannot carry the `Condition` blocks that would check them (see [Trust Policy Validation](#trust-policy-validation)). Do not rely on an external ID as a security boundary.
+- `--serial-number` / `--token-code` (MFA) are not supported and return `InvalidParameterValue`.
+- `--policy` / `--policy-arns` (session policies) are not supported and return `PackedPolicyTooLarge`.
+
+## Using Temporary Credentials
+
+Temporary credentials are used exactly like access keys, plus a third value — the session token. Export all three as environment variables (which take precedence over any profile):
+
+```bash
+export AWS_ACCESS_KEY_ID=ASIA1A2B3C4D5E6F7890
+export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+export AWS_SESSION_TOKEN=IQoJb3JpZ2luX2VjE...
+export AWS_ENDPOINT_URL=https://localhost:9999
+export AWS_CA_BUNDLE=/var/lib/spinifex/config/ca.pem
+export AWS_DEFAULT_REGION=ap-southeast-2
+```
+
+Or store them in a named profile — the same shape as a user profile with `aws_session_token` added:
+
+```bash
+aws configure set aws_access_key_id ASIA1A2B3C4D5E6F7890 --profile spinifex-deploy
+aws configure set aws_secret_access_key wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY --profile spinifex-deploy
+aws configure set aws_session_token IQoJb3JpZ2luX2VjE... --profile spinifex-deploy
+aws configure set region ap-southeast-2 --profile spinifex-deploy
+aws configure set endpoint_url https://localhost:9999 --profile spinifex-deploy
+aws configure set ca_bundle /var/lib/spinifex/config/ca.pem --profile spinifex-deploy
+```
+
+Requests are then authorised as the assumed role, not your user:
+
+```bash
+aws sts get-caller-identity
+# "Arn": "arn:aws:sts::000000000001:assumed-role/deploy/release-42"
+
+aws ec2 describe-instances     # allowed by the role's ec2-read policy
+aws iam list-users             # AccessDenied — the role grants ec2:Describe* only
+```
+
+When the expiration time passes, every call fails until you fetch fresh credentials — there is no refresh; run `assume-role` again.
+
+## Session Tokens for Your Own User
+
+`get-session-token` issues temporary credentials with your **own user's** permissions — no role, no trust policy. Useful for handing a build script credentials that expire on their own instead of your long-lived access key:
+
+```bash
+aws sts get-session-token --duration-seconds 3600
+```
+
+```json
+{
+    "Credentials": {
+        "AccessKeyId": "ASIA1A2B3C4D5E6F7890",
+        "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+        "SessionToken": "IQoJb3JpZ2luX2VjE...",
+        "Expiration": "2026-07-03T11:00:00Z"
+    }
+}
+```
+
+Duration runs from **900** to **129600** seconds (36 hours), defaulting to **43200** (12 hours). Unlike `assume-role`, out-of-range values are **clamped, not rejected** — asking for 200000 seconds quietly returns a 36-hour session. Use the credentials exactly as in [Using Temporary Credentials](#using-temporary-credentials); `get-caller-identity` keeps reporting your user ARN, since no role is involved.
+
+MFA (`--serial-number` / `--token-code`) is not supported and returns `InvalidParameterValue`.
+
+## Web Identity Federation (IRSA)
+
+`assume-role-with-web-identity` exchanges an OIDC ID token for role credentials — the mechanism behind **IRSA** (IAM Roles for Service Accounts), where a Kubernetes pod's ServiceAccount token becomes its cloud identity. The call is anonymous: the JWT is the identity, so no IAM credentials sign the request.
+
+In Spinifex the token issuer is an [EKS](/docs/eks) cluster. Each cluster publishes a signing-key set (JWKS) under an issuer URL of the form `https://{host}/oidc/eks/{region}/{account-id}/{cluster-name}`, and STS verifies tokens against it directly — no external identity provider is contacted.
+
+### Register the OIDC Provider
+
+The role account must register the issuer as an IAM OIDC provider before any token from it is accepted:
+
+```bash
+aws iam create-open-id-connect-provider \
+  --url https://spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod \
+  --client-id-list sts.amazonaws.com \
+  --thumbprint-list 9e99a48a9960b14926bb7f3b02e22da2b0ab7280
+```
+
+```json
+{
+    "OpenIDConnectProviderArn": "arn:aws:iam::000000000001:oidc-provider/spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod",
+    "Tags": []
+}
+```
+
+Manage providers with the usual verbs — `get-open-id-connect-provider` and `list-open-id-connect-provider-tags` take the ARN, and tags work like every other IAM resource:
+
+```bash
+aws iam list-open-id-connect-providers
+aws iam get-open-id-connect-provider \
+  --open-id-connect-provider-arn arn:aws:iam::000000000001:oidc-provider/spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod
+aws iam delete-open-id-connect-provider \
+  --open-id-connect-provider-arn arn:aws:iam::000000000001:oidc-provider/spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod
+```
+
+### The IRSA Trust Policy
+
+The role's trust policy names the provider as a `Federated` principal and pins the token's `sub` (the ServiceAccount) and `aud` claims with `StringEquals` — the one place a `Condition` block is allowed:
+
+```bash
+cat > /tmp/irsa-trust.json << 'EOF'
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws:iam::000000000001:oidc-provider/spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod"
+      },
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod:sub": "system:serviceaccount:default:app",
+          "spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod:aud": "sts.amazonaws.com"
+        }
+      }
+    }
+  ]
+}
+EOF
+
+aws iam create-role --role-name irsa-app \
+  --assume-role-policy-document file:///tmp/irsa-trust.json
+```
+
+The condition keys are the issuer URL (scheme stripped) followed by `:sub` or `:aud`. Only these two keys and only `StringEquals` are accepted — anything wider (`StringLike`, wildcards, other keys) is rejected at `create-role` with `MalformedPolicyDocument` rather than silently over-granting.
+
+### Exchange the Token
+
+```bash
+aws sts assume-role-with-web-identity \
+  --role-arn arn:aws:iam::000000000001:role/irsa-app \
+  --role-session-name pod-app-7f9c4 \
+  --web-identity-token "$(cat /var/run/secrets/eks.amazonaws.com/serviceaccount/token)"
+```
+
+The response has the same `Credentials` block as `assume-role`, plus the verified `SubjectFromWebIdentityToken`, `Provider`, and `Audience`. Duration runs 900–43200 seconds (default 3600), independent of the role's `MaxSessionDuration`.
+
+For a token to be accepted it must be an ES256-signed JWT whose `iss` matches a registered OIDC provider in the role's account, whose `aud` contains `sts.amazonaws.com`, whose signature verifies against the cluster's published JWKS, and which has not expired — any failure returns `InvalidIdentityToken`. In-cluster, none of this is manual: pods with an annotated ServiceAccount get the token mounted and the AWS SDK performs the exchange automatically.
+
+## Trust Policy Validation
+
+Spinifex validates trust policies (`AssumeRolePolicyDocument`) more strictly than AWS, rejecting at write time with `MalformedPolicyDocument`:
+
+- `Condition` blocks anywhere **except** the `StringEquals` IRSA form on `sts:AssumeRoleWithWebIdentity` described above. This is why `--external-id` cannot be enforced.
+- `NotPrincipal` and `NotAction` elements.
+- Empty `Principal` blocks and empty-string `Action` elements.
+
+Everything the validator accepts is evaluated at assume time; explicit `Deny` statements win, and a policy that matches no statement denies.
+
+## Troubleshooting
+
+### AccessDenied When Assuming a Role
+
+The role's trust policy does not allow your principal, or the role does not exist — a missing role is deliberately indistinguishable from a denied one, matching AWS. Check who you are, that the role exists, and what it trusts:
+
+```bash
+aws sts get-caller-identity
+aws iam list-roles --query 'Roles[].Arn'
+aws iam get-role --role-name deploy \
+  --query 'Role.AssumeRolePolicyDocument'
+```
+
+For web identity, `AccessDenied` also covers a trust-policy `Condition` that doesn't match the token's `sub`/`aud` claims.
+
+### ValidationError on Duration
+
+The requested `--duration-seconds` is outside 900–min(`MaxSessionDuration`, 43200). Check and raise the role's ceiling:
+
+```bash
+aws iam get-role --role-name deploy --query 'Role.MaxSessionDuration'
+aws iam update-role --role-name deploy --max-session-duration 43200
+```
+
+Note `get-session-token` never returns this — it clamps instead.
+
+### InvalidParameterValue
+
+You passed an MFA flag (`--serial-number` / `--token-code`) — MFA is not supported. `--tags` and `--transitive-tag-keys` on `assume-role` return the same error.
+
+### PackedPolicyTooLarge
+
+You passed `--policy` or `--policy-arns`. Session policies are not supported — scope the role's own permissions instead.
+
+### InvalidIdentityToken
+
+The web identity token failed verification: malformed JWT, wrong signature algorithm (only ES256 is accepted), expired, `aud` missing `sts.amazonaws.com`, issuer not registered as an OIDC provider in the role's account, or the cluster's JWKS is unavailable. Confirm the provider is registered:
+
+```bash
+aws iam list-open-id-connect-providers
+```
+
+### Credentials Suddenly Rejected
+
+Temporary credentials expired. Check the `Expiration` from the original response and mint a fresh set — sessions cannot be renewed or extended.
@@ -97,8 +97,8 @@ iface br-wan inet dhcp

 Walkthrough of changes:

-* Adds eno3 into the bridge
-* eno3 becomes a slave interface
+* Adds enp0s1 into the bridge
+* enp0s1 becomes a slave interface
 * All traffic goes through br-wan
 * Spinifex EC2 (VMs) in a public subnet, obtain an IP address from the WAN upstream

@@ -1,7 +1,7 @@
 ---
 title: "Air-Gapped Install"
 description: "Deploy Spinifex in environments without internet connectivity. Covers using a pre-built release tarball, mirrored APT packages, and locally-staged cloud images."
-category: "Getting Started"
+category: "Install"
 tags:
  - install
  - air-gapped
@@ -1,7 +1,7 @@
 ---
 title: "Multi-Node Install"
 description: "Deploy Spinifex across multiple servers to create an availability zone with high availability, data durability, and fault tolerance."
-category: "Getting Started"
+category: "Install"
 tags:
  - install
  - multi node
@@ -45,7 +45,7 @@ A Spinifex cluster distributes services across multiple servers for high availab
 >
 > Before running the installer on any server, that server's WAN interface **must** already be enslaved to a Linux bridge named `br-wan`. The host IP, default route, and DHCP must all live on the bridge — not on the bare NIC.
 >
-> The bootstrap installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in the next release.
+> The bootstrap installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in a future release.
 >
 > **Verify on every node before continuing:**
 >
@@ -1,7 +1,7 @@
 ---
 title: "Source Install"
 description: "Build Spinifex from source for development, custom builds, or contributing."
-category: "Getting Started"
+category: "Install"
 tags:
  - install
  - source
@@ -29,12 +29,10 @@ resources:

 This guide builds Spinifex from source. For production deployments, the [binary installer](/docs/install) is recommended.

-**Requirements:**
+**Supported Operating Systems:**

- Ubuntu 26.04 / Debian 13
- Go 1.26.4+
- GCC, make, pkg-config
- QEMU/KVM, OVN/Open vSwitch, AWS CLI v2, dhcpcd-base
+- Ubuntu 26.04
+- Debian 13

 ## Instructions

@@ -47,13 +45,15 @@ sudo make -C spinifex quickinstall
 export PATH=$PATH:/usr/local/go/bin
 ```

-## Step 2. Clone and Build
+## Step 2. Clone Sibling Repositories

 ```bash
 cd spinifex
 ./scripts/clone-deps.sh
 ```

+This clones Predastore (S3) and Viperblock (EBS) alongside Spinifex.
+
 ## Step 3. Development Initialisation

 ```bash
@@ -1,7 +1,7 @@
 ---
 title: "Bootable USB Install"
 description: "Install Spinifex on bare-metal hardware by flashing the Spinifex ISO to a USB drive and booting the target device from it."
-category: "Getting Started"
+category: "Install"
 sections:
  - overview
  - troubleshooting
@@ -1,7 +1,7 @@
 ---
 title: "Single-Node Install"
 description: "Install Spinifex on a single server using the binary installer."
-category: "Getting Started"
+category: "Install"
 tags:
  - install
  - single node
@@ -55,7 +55,7 @@ This guide installs Spinifex on a single server using the binary installer. For
 >
 > Before running the installer, the host's WAN interface **must** already be enslaved to a Linux bridge named `br-wan`. The host IP, default route, and DHCP must all live on the bridge — not on the bare NIC.
 >
-> The bootstrap installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in the next release.
+> The bootstrap installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in a future release.
 >
 > **Verify before continuing:**
 >
@@ -31,7 +31,8 @@ Spinifex's hybrid mode enables bidirectional data synchronization between local

 ## Prerequisites

-Ensure the AWS profile is set:
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:

 ```bash
 export AWS_PROFILE=spinifex
@@ -31,15 +31,16 @@ resources:

 Spinifex provides drop-in compatibility with AWS APIs, making it possible to migrate existing workloads with minimal changes.

-**Supported Services:** VPC, EC2, EBS, S3, IAM
+**Supported Services:** EC2, VPC, EBS, S3, IAM, STS, ELBv2 (ALB/NLB), ACM, ECR, ECS, EKS

-**Compatible Tools:** AWS CLI, AWS SDKs, Terraform, any S3-compatible client
+**Compatible Tools:** AWS CLI, AWS SDKs, Terraform, kubectl (via `aws eks get-token`), any S3-compatible client

 ## Instructions

 ## Prerequisites

-Ensure the AWS profile is set:
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:

 ```bash
 export AWS_PROFILE=spinifex
@@ -61,13 +62,21 @@ provider "aws" {
  secret_key = "your-spinifex-secret-key"

  endpoints {
-    ec2 = "https://localhost:9999"
-    s3  = "https://localhost:8443"
+    ec2                = "https://localhost:9999"
+    iam                = "https://localhost:9999"
+    sts                = "https://localhost:9999"
+    elbv2              = "https://localhost:9999"
+    acm                = "https://localhost:9999"
+    ecr                = "https://localhost:9999"
+    ecs                = "https://localhost:9999"
+    eks                = "https://localhost:9999"
+    s3                 = "https://localhost:8443"
  }

  skip_credentials_validation = true
  skip_metadata_api_check     = true
  skip_requesting_account_id  = true
+  skip_region_validation      = true
 }
 ```

@@ -81,12 +90,13 @@ aws s3 sync s3://local-bucket/ s3://cloud-bucket/ --source-region spinifex --reg

 ## Terraform Provider Errors

-Ensure all three skip flags are set in your provider configuration:
+Ensure all four skip flags are set in your provider configuration:

 ```hcl
 skip_credentials_validation = true
 skip_metadata_api_check     = true
 skip_requesting_account_id  = true
+skip_region_validation      = true
 ```

 Without these, Terraform will try to validate credentials and metadata against real AWS endpoints.
@@ -266,7 +266,7 @@ aideinit
 mv /var/lib/aide/aide.db.new /var/lib/aide/aide.db
 ```

-Daily check — a systemd timer invoking `aide --check`. Non-zero exit must alert. Re-baseline after every Spinifex upgrade (`make reinstall`, package upgrade, config change) and record the new hash.
+Daily check — a systemd timer invoking `aide --check`. Non-zero exit must alert. Re-baseline after every Spinifex upgrade, package upgrade, or config change and record the new hash.

 ### 4.3 Wazuh (centralised AV + FIM + alerting)

@@ -77,7 +77,7 @@ Any medium in any of these roles on a Spinifex node has held, or may have held,
 | Viperblock chunk cache / local backing | NVMe/SATA SSD or HDD | Encrypted volume chunks. |
 | Predastore object store | NVMe/SATA SSD or HDD | All S3 object data — AMIs, snapshots, user-uploaded artifacts, IAM state files, tenant data. |
 | NATS JetStream disk | NVMe/SATA SSD | IAM NATS token, cluster metadata, pending job state, NATS KV entries (including wrapped DEKs). |
-| Removable master-key media | USB flash token, HSM, smart card | Master encryption key (Phase 4 USB-mount path). Destroying this sanitizes every DEK-encrypted volume cluster-wide. |
+| Removable master-key media | USB flash token, HSM, smart card | Master encryption key (USB-mount path). Destroying this sanitizes every DEK-encrypted volume cluster-wide. |
 | Backup media | LTO tape, removable HDD/SSD, off-site cloud backup | Any of the above, point-in-time. |
 | BMC / iDRAC / iLO storage | Embedded flash | Console recordings, SEL logs, BMC credentials, cached operator certificates. Sanitize via the BMC "Reset to defaults" / "Erase user data" command before disposal. |
 | Switch / router config storage | Embedded flash | Cluster VLAN, management IPs, ACLs. Operator network kit, out of Spinifex scope but noted for completeness. |
@@ -193,7 +193,7 @@ Third-party destruction vendors must provide a certificate of destruction listin
 When retiring a whole node:

 1. **Drain** — migrate or terminate all instances hosted on the node; confirm Predastore and Viperblock roles are reassigned if the node held one. Volume DEKs of terminated instances are already cryptographically erased per [§3.1](#31-cryptographic-erase-via-dek-deletion).
-2. **Deregister** — `bd`-linked change ticket; the operator removes the node from the cluster (`spx admin node remove`) so NATS routes, OVN chassis, and predastore distributed membership drop it.
+2. **Deregister** — raise a change ticket; the operator removes the node from the cluster (`spx admin node remove`) so NATS routes, OVN chassis, and predastore distributed membership drop it.
 3. **Stop services and unmount** — `systemctl stop 'spinifex-*'`, `nbdkit` sessions, OVN agent. Confirm `/run/spinifex/nbd/` is empty.
 4. **Destroy on-disk keys** — see [§5](#5-key-destruction). This is the single most important step: sanitization of volume data reduces to sanitization of keys once data is encrypted.
 5. **Sanitize each drive** by media type per [§4.1](#41-nvme-ssd) – [§4.3](#43-hdd). Every drive bay is sanitized, including any unpopulated cache or WAL devices.
@@ -211,7 +211,7 @@ Cryptographic erase relies on the key being unrecoverable. A drive that previous
 | Cluster CA private key (`/etc/spinifex/ca.key`) | Leader node system disk | Same treatment. Loss of the CA key does not sanitize data, but prevents impersonation of the cluster identity post-disposal. |
 | Per-node TLS private key (`/etc/spinifex/server.key`) | Each node system disk | `shred -u` before drive sanitization. |
 | Wrapped DEKs in NATS KV | `/var/lib/spinifex/nats/` on NATS-carrying nodes | Deleted via the `DeleteVolume` API; the KV record is compacted. Sanitization of the NATS JetStream disk per [§4](#4-whole-drive-and-node-decommissioning--before-disposal-mpl1-383-a) ensures no recoverable copies. |
-| Removable master-key media (Phase 4 USB) | USB flash token | Physically destroy the token when the cluster is decommissioned; alternatively retain inside a secured location as evidence of cluster-wide cryptographic erase. Do not reuse for another cluster without full sanitization (see [§6](#6-removable-and-backup-media)). |
+| Removable master-key media (USB) | USB flash token | Physically destroy the token when the cluster is decommissioned; alternatively retain inside a secured location as evidence of cluster-wide cryptographic erase. Do not reuse for another cluster without full sanitization (see [§6](#6-removable-and-backup-media)). |
 | Backup-copy keys | Any off-site or escrow copy | Destroy simultaneously with the primary. A surviving backup defeats the cryptographic erase. |

 If even one copy of the master key survives and the encrypted drives are recoverable, cryptographic erase has **not** been achieved. Track every key copy in the device register ([Physical Security Guide §5](/docs/security/physical-security-guide#5-manage-physical-access-devices-pel1-3105)).
@@ -88,12 +88,14 @@ The inventory in [§1](#1-inbound-listeners)–[§2](#2-outbound-connections) sa
 | 8443 | spinifex-predastore | HTTPS | Cluster | S3-compatible object storage (AMIs, snapshots, user objects) | AWS SigV4 + TLS |
 | 6660–6662 | predastore (Raft) | TCP | Cluster | Metadata consensus (3 nodes) | Cluster network only |
 | 9991–9993 | predastore (data shards) | TCP | Cluster | Erasure-coded data shard transport | Cluster network only |
-| 6641 | OVN Northbound DB | OVSDB/TCP | Cluster | Logical network topology consumed by vpcd | Cluster network only; TLS planned |
-| 6642 | OVN Southbound DB | OVSDB/TCP | Cluster | Chassis / port / MAC binding state | Cluster network only; TLS planned |
+| 6641 | OVN Northbound DB (client) | OVSDB/TCP | Cluster | Logical network topology consumed by vpcd | Cluster network only; TLS planned |
+| 6642 | OVN Southbound DB (client) | OVSDB/TCP | Cluster | Chassis / port / MAC binding state | Cluster network only; TLS planned |
+| 6643 | OVN Northbound DB (RAFT) | OVSDB/TCP | Cluster | NB database RAFT replication between the 3 quorum nodes | Cluster network only; TLS planned |
+| 6644 | OVN Southbound DB (RAFT) | OVSDB/TCP | Cluster | SB database RAFT replication between the 3 quorum nodes | Cluster network only; TLS planned |
 | 8222 | spinifex-nats (monitoring) | HTTP | Localhost | `varz`/`subsz` metrics consumed by the daemon | Loopback only |
 | socket / dynamic TCP | nbdkit (Viperblock) | NBD | Host-local / cluster | Block device transport for guest EBS volumes | Unix socket by default; TCP only in remote/DPU mode |

-¹ **Formation port lifecycle.** 4432 opens during `spx admin init` / `spx admin join` while a bootstrap token is outstanding and closes once the cluster is formed (token TTL default 30 min, `--token-ttl`). The server presents an ephemeral self-signed cert that pre-dates trust bootstrap, so the joining node dials with `InsecureSkipVerify` — the only production dial that does. Authenticity rests on the operator supplying the leader address out-of-band plus possession of the bearer token. Document in the security plan so reviewers do not flag 4432 as a persistent open port.
+¹ **Formation port lifecycle.** 4432 opens during `spx admin init` / `spx admin join` while a bootstrap token is outstanding and closes once the cluster is formed (token TTL default 30 min, `--token-ttl`). The server presents an ephemeral self-signed cert that pre-dates trust bootstrap, so the joining node does not verify the certificate chain for this single dial. Authenticity rests on the operator supplying the leader address out-of-band plus possession of the bearer token. Document in the security plan so reviewers do not flag 4432 as a persistent open port.

 **Development-only listeners.** When `dev_networking=true`, QEMU opens arbitrary host TCP ports for SSH port-forwarding into guest VMs. Production installs (the `/etc/spinifex` layout) do not enable this; it must not appear on compliance nodes.

@@ -126,7 +128,8 @@ Control-plane and data-plane traffic between Spinifex nodes, for completeness an
 | Predastore S3 | 8443 | TLS + AWS SigV4 | Cross-node object reads/writes |
 | Predastore Raft | 6660–6662 | Cluster network only | Metadata consensus |
 | Predastore shards | 9991–9993 | Cluster network only | Erasure-coded data shards |
-| OVN NB/SB | 6641 / 6642 | Cluster network only (TLS planned) | Network control plane |
+| OVN NB/SB (client) | 6641 / 6642 | Cluster network only (TLS planned) | Network control plane; vpcd and ovn-controller dial the quorum |
+| OVN NB/SB (RAFT) | 6643 / 6644 | Cluster network only (TLS planned) | NB/SB database replication across the 3 quorum nodes |
 | OVN tunnels (Geneve) | UDP 6081 | None | Tenant traffic overlay between chassis, inside the cluster subnet |

 Nodes **must** sit on a network segment that is not routed to tenant/guest VMs or to the internet. Predastore Raft/shards and OVN DBs are cluster-internal and must not be reachable from anywhere else.
@@ -140,7 +143,7 @@ Default external surface is three listeners — **9999** (AWS API), **3000** (UI
 tcp dport { 22, 9999, 3000 } accept

 # Cluster-only: replace 10.0.1.0/24 with your cluster CIDR
-ip saddr 10.0.1.0/24 tcp dport { 4222, 4248, 8443, 6641, 6642, 6660-6662, 9991-9993 } accept
+ip saddr 10.0.1.0/24 tcp dport { 4222, 4248, 8443, 6641-6644, 6660-6662, 9991-9993 } accept
 ip saddr 10.0.1.0/24 udp dport 6081 accept

 # Default deny
@@ -162,7 +165,7 @@ Every listener and outbound destination is controlled by one of these files. Cha
 | OVN packages (`ovn-central`, `ovn-host`) | `ovn-nb-db`, `ovn-sb-db` (via `ovs-vsctl set open_vswitch …`) | OVN DB bind addresses. |
 | Spinifex UI service | Built-in defaults: `host = "0.0.0.0"`, `port = 3000`. No `spinifex.toml` block today. | UI listener. |
 | `spx admin init` / `spx admin join` | `--port`, `--token-ttl`, `--no-telemetry` (or `SPX_NO_TELEMETRY=1`) | Formation port, token TTL, telemetry opt-out. |
-| `utils/images.go` `AvailableImages` | Image catalogue URLs | Outbound HTTPS destinations for image downloads. |
+| Image catalogue (built-in) | Fixed URLs listed in [§2](#2-outbound-connections); not operator-configurable | Outbound HTTPS destinations for image downloads. |

 ## 6. Operator Checklist

@@ -172,6 +175,6 @@ Every listener and outbound destination is controlled by one of these files. Cha
 - Formation port 4432 is closed on nodes not actively running a bootstrap token.
 - Outbound HTTPS restricted to the [§2](#2-outbound-connections) image-catalogue hosts, or replaced with air-gapped import.
 - Install telemetry (`install.mulgadc.com`) is either permitted and recorded in the security plan, or disabled via `SPX_NO_TELEMETRY=1` / `--no-telemetry`.
- OVN NB/SB (6641/6642) exposure limited to the cluster subnet pending the L2 TLS work.
+- OVN NB/SB client and RAFT ports (6641–6644) exposure limited to the cluster subnet pending the L2 TLS work.
 - SSH (22) configured to operator-managed keys only; password auth disabled in `sshd_config`.
 - Periodic review (at least annually, and after any topology change) confirms this inventory still matches the deployed configuration.
@@ -300,6 +300,14 @@ suites:
    # spinifex_daemon. docker/crane/skopeo data-plane subtests skip when the
    # client binary or registry-host resolution is absent.

+  e2e-bedrock:
+    path: tests/e2e/bedrock
+    covers: [awsgw]
+    # Ochre (AWS-Bedrock) control plane: static catalog list/get served
+    # straight from awsgw (gateway/bedrock.go, gateway/bedrock/) — no NATS
+    # hop to spinifex_daemon. Converse (Anthropic-direct / self-host vLLM)
+    # subtests are gated on backend availability and skip by default.
+
  e2e-ecs:
    path: tests/e2e/ecs
    covers: [awsgw, spinifex_daemon, vpcd, viperblockd, predastore]
@@ -332,6 +340,12 @@ suites:
  # e2e-bm:
  #   path: spinifex/tests/e2e/baremetal
  #   covers: [awsgw, spinifex_daemon, vpcd, viperblockd, predastore]
+  #
+  # e2e-natuplink activates once CI can provision an external_mode=nat
+  # node (setup-ovn.sh --nat-uplink + spx admin init --external-mode=nat).
+  # e2e-natuplink:
+  #   path: spinifex/tests/e2e/natuplink
+  #   covers: [awsgw, spinifex_daemon, vpcd, viperblockd, predastore]


 # Special-case rules consumed by the selector (Bead 4). Hard-coded in
@@ -77,7 +77,7 @@ Terraform manages the cluster, the addons, the ACM cert, and the HTTPS Ingress;

 - The `argocd` and `aws-ebs-csi-driver` bundles must be baked into the `eks-node` AMI — `describe-addon` returns `no baked bundle` otherwise.
 - The EBS-CSI default StorageClass uses `provisioner: ebs.csi.aws.com`; a PVC against it provisions a **Viperblock-backed EBS volume**.
- **Verify dynamic provisioning on a live cluster.** EBS-CSI dynamic provisioning depends on the running Spinifex EBS/Viperblock data plane and may not be available on every build yet. If the PVC stays `Pending`, the app still serves (the counter falls back to in-memory) — confirm the volume binds on a stack where the driver is healthy.
+- If the PVC stays `Pending`, the app still serves (the counter falls back to in-memory) — see [Troubleshooting](#troubleshooting).
 - Leave addon versions unset (the AWS provider and the catalog disagree on the version-string format).
 - The Argo CD `Application` is a `kubernetes_manifest`, so the `argoproj.io` CRDs must exist before `apply` — apply the parent and let the addon reach `ACTIVE` first.

@@ -256,7 +256,7 @@ kubectl describe pvc <name>
 kubectl get storageclass
 ```

-Confirm the `aws-ebs-csi-driver` addon is `ACTIVE` and the default StorageClass exists. EBS-CSI dynamic provisioning depends on the live Spinifex EBS/Viperblock data plane — verify on a stack where the driver is healthy. The app still serves with an in-memory counter if the volume never binds.
+Confirm the `aws-ebs-csi-driver` addon is `ACTIVE` and the default StorageClass exists. The app still serves with an in-memory counter if the volume never binds.

 ### The Ingress Has No Address

@@ -176,8 +176,6 @@ aws eks describe-addon --cluster-name eks-https --addon-name aws-load-balancer-c

 Confirm the addon is `ACTIVE` and the cluster carries `spinifex.io/managed-ingress = "false"` (otherwise the built-in ingress competes).

-> **Verify on a live cluster.** The LBC reconcile path (ALB creation, target registration, listener wiring) exercises the running Spinifex EKS data plane end to end. Stand this up on a live stack to confirm the ALB serves the page; the manifests here are the AWS-parity shape of that flow.
-
 ### HTTPS Loads but Returns 502 / 504

 The ALB reached the workers but the backend didn't answer. Check the demo pods and the NodePort SG rule:
@@ -1,36 +1,53 @@
 module github.com/mulgadc/spinifex

-go 1.26.4
+go 1.26.5

 require (
 	github.com/aws/aws-sdk-go v1.55.8
+	github.com/aws/aws-sdk-go-v2 v1.42.1
+	github.com/aws/aws-sdk-go-v2/config v1.32.29
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.28
+	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
-	github.com/containerd/containerd/v2 v2.3.2
+	github.com/containerd/containerd/v2 v2.3.3
 	github.com/distribution/reference v0.6.0
-	github.com/go-chi/chi/v5 v5.3.0
+	github.com/go-chi/chi/v5 v5.3.1
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/uuid v1.6.0
 	github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2
-	github.com/klauspost/cpuid/v2 v2.3.0
-	github.com/mulgadc/predastore v1.10.0
-	github.com/mulgadc/viperblock v1.10.0
-	github.com/nats-io/nats-server/v2 v2.14.2
+	github.com/klauspost/cpuid/v2 v2.4.0
+	github.com/mulgadc/predastore v1.12.0
+	github.com/mulgadc/viperblock v1.12.0
+	github.com/nats-io/nats-server/v2 v2.14.3
 	github.com/nats-io/nats.go v1.52.0
 	github.com/opencontainers/runtime-spec v1.3.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
-	github.com/pelletier/go-toml/v2 v2.4.2
+	github.com/pelletier/go-toml/v2 v2.4.3
 	github.com/pterm/pterm v0.12.83
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
+	go.opentelemetry.io/contrib/bridges/otelslog v0.19.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0
+	go.opentelemetry.io/contrib/instrumentation/runtime v0.69.0
+	go.opentelemetry.io/otel v1.44.0
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0
+	go.opentelemetry.io/otel/log v0.20.0
+	go.opentelemetry.io/otel/metric v1.44.0
+	go.opentelemetry.io/otel/sdk v1.44.0
+	go.opentelemetry.io/otel/sdk/log v0.20.0
+	go.opentelemetry.io/otel/sdk/metric v1.44.0
+	go.opentelemetry.io/otel/trace v1.44.0
 	go.uber.org/goleak v1.3.0
-	golang.org/x/crypto v0.53.0
-	golang.org/x/sync v0.21.0
-	golang.org/x/sys v0.46.0
+	go.yaml.in/yaml/v3 v3.0.4
+	golang.org/x/crypto v0.54.0
+	golang.org/x/sync v0.22.0
+	golang.org/x/sys v0.47.0
 	gopkg.in/ini.v1 v1.67.3
-	gopkg.in/yaml.v3 v3.0.1
 )

 require (
@@ -42,13 +59,22 @@ require (
 	github.com/antithesishq/antithesis-sdk-go v0.7.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.42.0 // indirect
-	github.com/aws/smithy-go v1.27.1 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.4.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.32.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.44.0 // indirect
+	github.com/aws/smithy-go v1.27.3 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/buraksezer/consistent v0.10.0 // indirect
-	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cenkalti/hub v1.0.2 // indirect
 	github.com/cenkalti/rpc2 v1.0.5 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
@@ -72,7 +98,7 @@ require (
 	github.com/containerd/ttrpc v1.2.8 // indirect
 	github.com/containerd/typeurl/v2 v2.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/dgraph-io/badger/v4 v4.9.2 // indirect
+	github.com/dgraph-io/badger/v4 v4.9.4 // indirect
 	github.com/dgraph-io/ristretto/v2 v2.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
@@ -91,6 +117,7 @@ require (
 	github.com/google/flatbuffers v25.12.19+incompatible // indirect
 	github.com/google/go-tpm v0.9.8 // indirect
 	github.com/gookit/color v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-metrics v0.5.4 // indirect
@@ -116,6 +143,7 @@ require (
 	github.com/minio/crc64nvme v1.1.1 // indirect
 	github.com/minio/highwayhash v1.0.4 // indirect
 	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/sys/capability v0.4.0 // indirect
 	github.com/moby/sys/mountinfo v0.7.2 // indirect
 	github.com/moby/sys/sequential v0.6.0 // indirect
 	github.com/moby/sys/signal v0.7.1 // indirect
@@ -130,6 +158,7 @@ require (
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -150,23 +179,25 @@ require (
 	go.etcd.io/bbolt v1.4.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
-	go.opentelemetry.io/otel v1.44.0 // indirect
-	go.opentelemetry.io/otel/metric v1.44.0 // indirect
-	go.opentelemetry.io/otel/trace v1.44.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
-	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/mod v0.36.0 // indirect
+	golang.org/x/mod v0.37.0 // indirect
 	golang.org/x/net v0.56.0 // indirect
-	golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 // indirect
-	golang.org/x/term v0.44.0 // indirect
-	golang.org/x/text v0.38.0 // indirect
+	golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect
+	golang.org/x/term v0.45.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
-	golang.org/x/tools v0.45.0 // indirect
+	golang.org/x/tools v0.47.0 // indirect
 	golang.org/x/vuln v1.1.4 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260618152121-87f3d3e198d3 // indirect
 	google.golang.org/grpc v1.81.1 // indirect
 	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
+	tags.cncf.io/container-device-interface v1.1.0 // indirect
+	tags.cncf.io/container-device-interface/specs-go v1.1.0 // indirect
 )

 // go.opencensus.io -> grpc@v1.33.2 drags in the pre-split monolithic genproto,
@@ -402,6 +402,8 @@ codeberg.org/go-latex/latex v0.0.1/go.mod h1:AiC91vVG2uURZRd4ZN1j3mAac0XBrLsxK6+
 codeberg.org/go-latex/latex v0.1.0/go.mod h1:LA0q/AyWIYrqVd+A9Upkgsb+IqPcmSTKc9Dny04MHMw=
 codeberg.org/go-pdf/fpdf v0.10.0/go.mod h1:Y0DGRAdZ0OmnZPvjbMp/1bYxmIPxm0ws4tfoPOc4LjU=
 contrib.go.opencensus.io/exporter/stackdriver v0.13.15-0.20230702191903-2de6d2748484/go.mod h1:uxw+4/0SiKbbVSD/F2tk5pJTdVcfIBBcsQ8gwcu4X+E=
+cyphar.com/go-pathrs v0.2.1 h1:9nx1vOgwVvX1mNBWDu93+vaceedpbsDqo+XuBGL40b8=
+cyphar.com/go-pathrs v0.2.1/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20221208032759-85de2813cf6b/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
@@ -522,66 +524,78 @@ github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZw
 github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
 github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo=
 github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
-github.com/aws/aws-sdk-go-v2 v1.42.0 h1:XvXMJTkFQtpBKIWZnmr9ZEOc2InWM2yldjXEJ/bymhA=
-github.com/aws/aws-sdk-go-v2 v1.42.0/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 h1:p1BBrg/Hhp6uK7zpejeI8QFXHJeC/mynzi04Sl03k9g=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13/go.mod h1:8cIfkE9MDhkRZGpQ22aV6/lkYeYSozpz16Smrs5x4Ls=
+github.com/aws/aws-sdk-go-v2 v1.42.1 h1:9eOTgu1z/dVtYpNZ3/8/XbbaX0x/BqE3HUzAzs6K0ek=
+github.com/aws/aws-sdk-go-v2 v1.42.1/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E=
 github.com/aws/aws-sdk-go-v2/config v1.18.14/go.mod h1:0pI6JQBHKwd0JnwAZS3VCapLKMO++UL2BOkWwyyzTnA=
 github.com/aws/aws-sdk-go-v2/config v1.27.4/go.mod h1:zq2FFXK3A416kiukwpsd+rD4ny6JC7QSkp4QdN1Mp2g=
 github.com/aws/aws-sdk-go-v2/config v1.29.12/go.mod h1:xse1YTjmORlb/6fhkWi8qJh3cvZi4JoVNhc+NbJt4kI=
+github.com/aws/aws-sdk-go-v2/config v1.32.29 h1:BcMHHnpiWKogf+gGfpj3K1w+Sktz29XDo/cPSAPO3FU=
+github.com/aws/aws-sdk-go-v2/config v1.32.29/go.mod h1:+Kbhn8Es4kPUph3F/0W7avykytc+Jh2Ld9/msv9ljV4=
 github.com/aws/aws-sdk-go-v2/credentials v1.13.14/go.mod h1:85ckagDuzdIOnZRwws1eLKnymJs3ZM1QwVC1XcuNGOY=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.4/go.mod h1:+30tpwrkOgvkJL1rUZuRLoxcJwtI/OkeBLYnHxJtVe0=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.65/go.mod h1:4zyjAuGOdikpNYiSGpsGz8hLGmUzlY8pc8r9QQ/RXYQ=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.24 h1:2hQqYCV9yqyePQ9o6dCrZc/zO8U3TwPr9mIKlZnPu/I=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.24/go.mod h1:IDwpACtwqHLISdzfwUUNq4P9DsB/h5BLg4FwJPNfqFY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.28 h1:zTXJSsNcoO91/mTXsZoYf0AK8dvNPiA58/VtyGXR+wM=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.28/go.mod h1:Kd9E0JzDBW/q1xbsHFrev/GnbAf5J0Ng8xoyc7HZ91Q=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 h1:/hi1JADLEW9YYryEz1w4GQu0EtP23pP553Cf9KgsDV4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30/go.mod h1:/3AOgy4K17Dm4ucMZVC/MJkzy5kmfKUcINRHZyo0koQ=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2/go.mod h1:wRQv0nN6v9wDXuWThpovGQjqF1HFdcgWjporw14lS8k=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 h1:f3vKqSo13fhTYb+JEcXwXefZQE26I1FB5eTSniU67ko=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29/go.mod h1:MzoLFUArKGpGD+ukmPiTPG1X5x4o6M2kq4v2dr1FiEc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 h1:xM/Is9cKMHa8Jj8zkvWhvrFkZsXJV9E+BB4g0HW0duQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30/go.mod h1:WueJeNDZvK1fMYEWJIkcivBfEzUkTpBhzlrUKKY8EuA=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2/go.mod h1:tyF5sKccmDz0Bv4NrstEr+/9YkSPJHrcO7UsUKf7pWM=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 h1:RdwIf/CuUsvJX3RgJagbOyotl/cxoLY4xviKuE7p2GY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29/go.mod h1:71wt8W2EgswdZy9Mf9KNnzxZ3TiZlv4caKghPktDOkA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 h1:jn46zC9LdsVR/ZpMIJqMqb8hHv31BlLx3ulVqNspUOk=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30/go.mod h1:1hTMsAgbdS/AtUi4bw8+gUuh1pceo+eXRLfpSuSQj3M=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 h1:VTGy885W5DKBxWRUJbym9hytNaYzsyaPkCHGRRMAOhU=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30/go.mod h1:AS0HycUvJRFvTt613AYDOgO2jzw+00cVSMny8XB3yMY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 h1:3GUprIsfmGcC5SACIyB0e7E0BM1O1b3Erl5CePYIAeQ=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31/go.mod h1:7PuV1yl5e2xnUbm+RqvVg5i2iBM8EyijZNoI9wsOoOc=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 h1:ZD2+BSw9vFsNlKYIasSNt3uDbjqqXIBcM13UJv/Lx2k=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12/go.mod h1:Ms4zlcVBbXbiP7EVLhl+lgjvA/a7YphqQ3Ih3174EmI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22 h1:V51LGlOq/1VsDsHUdoklAQi7rMmx4qQubvFYAlP2254=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22/go.mod h1:4Pzhyz8hJOm2bepgl+NjvRx8vlUFAIIvJnZ/MkcNPpU=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 h1:9Fjh6fi/U5JEStVZijmaMpUwE/gvBJj7x2B/PjbO9To=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23/go.mod h1:iMoT2f1tClxrWAAnKCXjZQ6LOmfLrMG14wmnWpM+F14=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 h1:DRebniUGZ2MqiiIVmQJ04vIXr918hubdHMnarSLEWyU=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29/go.mod h1:LfRkPCD8YHDM2E5eTkos2UpwYeZnBcVarTa8L59bJHA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 h1:hiME6pBzC7OTl9LMtlyTWBuEl1f4QBcUmFDKC7MLXtc=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29/go.mod h1:G7RP+uhagpKtKhd1BM9N6JQqjCcGEU47K5lBVZQyRQw=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0 h1:ta8csKy5vN91F3i5gGR85lFV0srBqySEji7Jroes6rE=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0/go.mod h1:77ZAgynvx1txMvDG8gGWoWkO1augYDxkp9JElWFgjQU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31/go.mod h1:I/1+z0VwL1GhQyLgkoHDlygpUZ+iTAwOQ/NsftiUL2I=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0 h1:XptwLL+UHXgafYMIHTy59IRovLbhz3znkxY2uS/pbXU=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do=
+github.com/aws/aws-sdk-go-v2/service/signin v1.4.0 h1:sLzmJGCMv+C8KqiJgEqDLB6vxaJGmobRh4rr//ZpA3w=
+github.com/aws/aws-sdk-go-v2/service/signin v1.4.0/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.3/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw=
 github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8=
 github.com/aws/aws-sdk-go-v2/service/sso v1.25.2/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.32.0 h1:qjMmry/cBDee1E/2gyvel0uRYCi3mwRZ2hf6N+GAodo=
+github.com/aws/aws-sdk-go-v2/service/sso v1.32.0/go.mod h1:u8af9Nqkmqnr96f7v9nHqzZT9XBwbXEkTiqT4ROuJSE=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.3/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.0 h1:fpOlDPI55HdszaxapEGk6HsGosOUaM2YPWJpjMgp8UI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.0/go.mod h1:DMPWJBjYs6+3+f/qhBFEFPPlQ6NlhWjai3dJNvipJ84=
 github.com/aws/aws-sdk-go-v2/service/sts v1.18.4/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s=
 github.com/aws/aws-sdk-go-v2/service/sts v1.28.1/go.mod h1:uQ7YYKZt3adCRrdCBREm1CD3efFLOUNH77MrUCvx5oA=
 github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
+github.com/aws/aws-sdk-go-v2/service/sts v1.44.0 h1:bLZ0PolJ8J+HkJHztcXORUpHXBye2U8298lCEMi6ZCU=
+github.com/aws/aws-sdk-go-v2/service/sts v1.44.0/go.mod h1:9gdl4RrflIdpDb2TlXshWgR1F9TeCkvqDx77Vpr4Z/Q=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
-github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8=
-github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY=
+github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/bazelbuild/rules_go v0.49.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs=
@@ -590,6 +604,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
@@ -601,6 +617,8 @@ github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cenkalti/hub v1.0.2 h1:Nqv9TNaA9boeO2wQFW8o87BY3zKthtnzXmWGmJqhAV8=
 github.com/cenkalti/hub v1.0.2/go.mod h1:8LAFAZcCasb83vfxatMUnZHRoQcffho2ELpHb+kaTJU=
 github.com/cenkalti/rpc2 v1.0.5 h1:T6l4SS3ja3eaJfRyZrn7Oco/PSx/pr3YK5cjCgLVLTk=
@@ -683,8 +701,8 @@ github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/q
 github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
 github.com/containerd/containerd/api v1.11.1 h1:h8nfoDW9+fNsC/9TwiAHj8B1GzXKtR4eFtkhi/X5RLU=
 github.com/containerd/containerd/api v1.11.1/go.mod h1:CaQFRu+N1MtbgL6JDOJLUB1hCKESU1lD6MuTJhgtdlw=
-github.com/containerd/containerd/v2 v2.3.2 h1:eLven1YxRMkeiKu7IcMrPKE+gn8sGR1DqHbbshMEvWM=
-github.com/containerd/containerd/v2 v2.3.2/go.mod h1:rHKGm3VW6wNrINb3x8mNT+w7qYXFVElTt/8HTuxVhD4=
+github.com/containerd/containerd/v2 v2.3.3 h1:MUNBVVBTBpPll7KPh5GTvkC3cfG03PQLAHVdsUoue9k=
+github.com/containerd/containerd/v2 v2.3.3/go.mod h1:rHKGm3VW6wNrINb3x8mNT+w7qYXFVElTt/8HTuxVhD4=
 github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg=
 github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
 github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
@@ -711,12 +729,14 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/cyphar/filepath-securejoin v0.6.0 h1:BtGB77njd6SVO6VztOHfPxKitJvd/VPT+OFBFMOi1Is=
+github.com/cyphar/filepath-securejoin v0.6.0/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgraph-io/badger/v4 v4.9.2 h1:Wb5qw8gElqwV1a8msHTeQKova9b1V10heFKMIiPd80E=
-github.com/dgraph-io/badger/v4 v4.9.2/go.mod h1:nJjaJTUOSsQEBhsq209FmwCvMJzEA3e74RjZw6V2pQI=
+github.com/dgraph-io/badger/v4 v4.9.4 h1:bcw+waCpzRZ2nmcSPbnPvDVhiEsn98TKmvnAhK7r7LM=
+github.com/dgraph-io/badger/v4 v4.9.4/go.mod h1:nJjaJTUOSsQEBhsq209FmwCvMJzEA3e74RjZw6V2pQI=
 github.com/dgraph-io/ristretto/v2 v2.4.0 h1:I/w09yLjhdcVD2QV192UJcq8dPBaAJb9pOuMyNy0XlU=
 github.com/dgraph-io/ristretto/v2 v2.4.0/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
@@ -799,8 +819,8 @@ github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8
 github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
 github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
-github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
+github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8=
+github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
 github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g=
 github.com/go-fonts/dejavu v0.3.2/go.mod h1:m+TzKY7ZEl09/a17t1593E4VYW8L1VaBXHzFZOIjGEY=
 github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks=
@@ -1083,10 +1103,14 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9K
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
 github.com/hamba/avro/v2 v2.17.2/go.mod h1:Q9YK+qxAhtVrNqOhwlZTATLgLA8qxG2vtvkhK8fJ7Jo=
 github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
 github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
@@ -1102,6 +1126,8 @@ github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP
 github.com/hashicorp/go-msgpack/v2 v2.1.5 h1:Ue879bPnutj/hXfmUk6s/jtIK90XxgiUIcXRl656T44=
 github.com/hashicorp/go-msgpack/v2 v2.1.5/go.mod h1:bjCsRXpZ7NsJdk45PoCQnzRGDaK8TKm5ZnDI/9y3J4M=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
@@ -1185,8 +1211,8 @@ github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuOb
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
-github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
-github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
+github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
+github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
 github.com/klauspost/reedsolomon v1.14.1 h1:swE9kzyWXD/wVG+l5Pe8bWnQ0giIY7D1GjCBKk3kG2U=
 github.com/klauspost/reedsolomon v1.14.1/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -1266,6 +1292,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
+github.com/moby/sys/capability v0.4.0 h1:4D4mI6KlNtWMCM1Z/K0i7RV1FkX+DBDHKVJpCndZoHk=
+github.com/moby/sys/capability v0.4.0/go.mod h1:4g9IK291rVkms3LKCDOoYlnV8xKwoDTpIrNEE35Wq0I=
 github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg=
 github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4=
 github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
@@ -1287,10 +1315,10 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/predastore v1.10.0 h1:kMBI31HTYV0fxXYY+XiLgxO6rJd4WPPDXswDgJaK8HY=
-github.com/mulgadc/predastore v1.10.0/go.mod h1:j2DOdAOj/c42MRI6SkZrfh9KFbIyDDJLmf63SJ2yizw=
-github.com/mulgadc/viperblock v1.10.0 h1:a8KKJ/qgeYsInlnOJ5S/o+HCfqTIklgzz/FpzdeXAEY=
-github.com/mulgadc/viperblock v1.10.0/go.mod h1:2ArCxLXJjSN4Idx9g8Pdq5n1Kp7HFHs2vWMZgKJT9dI=
+github.com/mulgadc/predastore v1.12.0 h1:JmoF//cIr3DgJhGypNVVLtaBZGCVUDzOo6z63407x54=
+github.com/mulgadc/predastore v1.12.0/go.mod h1:kgTncuCaB11e3eCQeZF0ydEFG0biuOlrZ4DlHNzwvZY=
+github.com/mulgadc/viperblock v1.12.0 h1:1PYG7GQ1cBbHbLI7QB7IzVu6EeKhrtcAJ+VAvNczjs8=
+github.com/mulgadc/viperblock v1.12.0/go.mod h1:5MGciKEvqpwLv2pdjk9j7PR8NHtXPs4pJzJi4UJojeU=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -1300,8 +1328,8 @@ github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL
 github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
 github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
 github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k=
-github.com/nats-io/nats-server/v2 v2.14.2 h1:Q7dRhCY03Y00rETFW3KV+KGaCIajlDfWgWUVgbMxyuk=
-github.com/nats-io/nats-server/v2 v2.14.2/go.mod h1:lWpb1bSpRELZfRdlMkdz8E7lbXKKyNe8RIn0vvepIHs=
+github.com/nats-io/nats-server/v2 v2.14.3 h1:+xjydPt7rkit67G+04TN0mcO2n+8nveZE7tK/PPV53A=
+github.com/nats-io/nats-server/v2 v2.14.3/go.mod h1:5IlCtBzfwyzQzPMjmoJ9W2/LKmnJRtNyuOs/OT+NHDY=
 github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
 github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc=
 github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
@@ -1324,6 +1352,10 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw
 github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
 github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg=
 github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116 h1:tAKu3NkKWZYpqBSOJKwTxT1wIGueiF7gcmcNgr5pNTY=
+github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116/go.mod h1:DKDEfzxvRkoQ6n9TGhxQgg2IM1lY4aM0eaQP4e3oElw=
+github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22F+ISDCJE=
+github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg=
 github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -1339,8 +1371,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
-github.com/pelletier/go-toml/v2 v2.4.2 h1:M2fKKbmyvI+hGId/D0W64qDBMVhJnNR10O5gIbMc//Q=
-github.com/pelletier/go-toml/v2 v2.4.2/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
+github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
 github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
@@ -1541,6 +1573,12 @@ github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 h1:tHNk7XK9GkmKUR6Gh8gV
 github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4=
 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@@ -1580,6 +1618,8 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/contrib/bridges/otelslog v0.19.0 h1:5RgvxieNq9tS3ewrV1vnODvbHPfKUIJcYtF9Cvz+6aQ=
+go.opentelemetry.io/contrib/bridges/otelslog v0.19.0/go.mod h1:iTBIdNwx/xmUhfgJs6+84S4dIK059811cO1eUBjKcHY=
 go.opentelemetry.io/contrib/detectors/gcp v1.28.0/go.mod h1:9BIqH22qyHWAiZxQh0whuJygro59z+nbMVuc7ciiGug=
 go.opentelemetry.io/contrib/detectors/gcp v1.29.0/go.mod h1:GW2aWZNwR2ZxDLdv8OyC2G8zkRoQBuURgV7RPQgcPoU=
 go.opentelemetry.io/contrib/detectors/gcp v1.31.0/go.mod h1:tzQL6E1l+iV44YFTkcAeNQqzXUiekSYP9jjJjXwEd00=
@@ -1615,8 +1655,10 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.69.0 h1:MtkMsuRo3zEXTTMALfyrszwCDZTkB6wolyPjbwFAdq0=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.69.0/go.mod h1:FYTxnpsm+UPD0erZNq20GvnM8T2YQHiHtT2vokdpoac=
 go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
 go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
 go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
@@ -1640,11 +1682,21 @@ go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOX
 go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
 go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
 go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0 h1:rydZ9sxbcFdm/oWrVyfLTjHIygMgv0bEeMd+3B/BvoM=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0/go.mod h1:earQ25dooT0Hhspq59DZ8YCC50jWfOlFEeWoxy/P444=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 h1:SUplec5dp06reu1zaXmOXdvqH398taqrDXqUl99jxSc=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0/go.mod h1:ho2g4N+ane+swq5I/VBkKWnRDY4kUINH3FuqyZqX/Ug=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo=
 go.opentelemetry.io/otel/exporters/prometheus v0.57.0/go.mod h1:QpFWz1QxqevfjwzYdbMb4Y1NnlJvqSGwyuU0B4iuc9c=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw=
+go.opentelemetry.io/otel/log v0.20.0 h1:/5i0vuHxCLWUfChWG41K9wkM0jafruPw9NU1/RCJirs=
+go.opentelemetry.io/otel/log v0.20.0/go.mod h1:wOcMcjsZpG8x7Bak7IhSi/lg8wscV2C1VdrKCLPlt0E=
 go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
 go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
 go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
@@ -1668,6 +1720,7 @@ go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9k
 go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
 go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
 go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
+go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA=
 go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk=
 go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
 go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
@@ -1691,6 +1744,10 @@ go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWm
 go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
 go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
 go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
+go.opentelemetry.io/otel/sdk/log v0.20.0 h1:vM3xI7TQgKPiSghe6urZtAkyFY7SodrSpC83CffDFuY=
+go.opentelemetry.io/otel/sdk/log v0.20.0/go.mod h1:Knej2nmsTUzN79T2eeXdRsjjPcoxoq2pUyUHz9TFyyU=
+go.opentelemetry.io/otel/sdk/log/logtest v0.20.0 h1:OqdRZ1guyzamK3M6LlRsmGqRrjkHWw6WZOKKli5ELpg=
+go.opentelemetry.io/otel/sdk/log/logtest v0.20.0/go.mod h1:PuMIlm7zAt7c3z8zfOI5ox4iT1Z87We+PF6YoINux/M=
 go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg=
 go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ=
 go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y=
@@ -1737,6 +1794,8 @@ go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v8
 go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
 go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
 go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
+go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
+go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
@@ -1815,8 +1874,8 @@ golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVo
 golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
 golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
 golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
-golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
-golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
+golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
+golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1928,8 +1987,8 @@ golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
 golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
 golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
 golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
-golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
-golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2135,8 +2194,9 @@ golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
-golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
 golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -2279,8 +2339,8 @@ golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
-golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
 golang.org/x/telemetry v0.0.0-20250710130107-8d8967aff50b/go.mod h1:4ZwOYna0/zsOKwuR5X/m0QFOJpSZvAxFfkQT+Erd9D4=
@@ -2293,8 +2353,8 @@ golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3c
 golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548=
 golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw=
 golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE=
-golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 h1:HjU6IWBiAgRIdAJ9/y1rwCn+UELEmwV+VsTLzj/W4sE=
-golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6/go.mod h1:Eqhaxk/wZsWEH8CRxLwj6xzEJbz7k1EFGqx7nyCoabE=
+golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc=
+golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2342,8 +2402,8 @@ golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
 golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
 golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
 golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
-golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
-golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2388,8 +2448,8 @@ golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
 golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
 golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
-golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
-golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
+golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2504,8 +2564,8 @@ golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg
 golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
 golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
 golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
-golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
-golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
 golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
 golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM=
@@ -2755,6 +2815,7 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.
 google.golang.org/genproto/googleapis/api v0.0.0-20260511170946-3700d4141b60/go.mod h1:7yoXV7RIh5gblj/xVYoogxAWvA9wUeVbpsK/M694l00=
 google.golang.org/genproto/googleapis/api v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:1dCETSCY2YKZNXQE3h4fun3TYwF5p8jejRKZgfWAgAY=
 google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
+google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3 h1:ctPmKL12ZsoKAlmPUsoW70zEDiYF+/H6aLieXxgAU0k=
 google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3/go.mod h1:Z4WJ5pJOYWFWcHEQUelD5QaZDknIQkpIL/+fyJOT9+A=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:+34luvCflYKiKylNwGJfn9cFBbcL/WrkciMmDmsTQ/A=
@@ -3154,4 +3215,10 @@ rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
 sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
+tags.cncf.io/container-device-interface v1.1.0 h1:RnxNhxF1JOu6CJUVpetTYvrXHdxw9j9jFYgZpI+anSY=
+tags.cncf.io/container-device-interface v1.1.0/go.mod h1:76Oj0Yqp9FwTx/pySDc8Bxjpg+VqXfDb50cKAXVJ34Q=
+tags.cncf.io/container-device-interface/specs-go v1.1.0 h1:QRZVeAceQM+zTZe12eyfuJuuzp524EKYwhmvLd+h+yQ=
+tags.cncf.io/container-device-interface/specs-go v1.1.0/go.mod h1:u86hoFWqnh3hWz3esofRFKbI261bUlvUfLKGrDhJkgQ=
--- a/Show More
+++ b/Show More