release: update go.mod for v1.11.0

A fresh cluster or SB re-bootstrap could leave a node's ovn-controller with
its OVSDB client latched on stale Southbound RAFT cluster state, looping on
"clustered database server has stale data" and silently no longer realising
new Port_Bindings. Cross-host CP data-NICs then never bind: no DHCP, no IMDS,
degraded boot. A manual `systemctl restart ovn-controller` was the only
recovery. Root cause is deterministic: setup-ovn.sh restarts ovn-controller
per node as each RAFT member forms, so a controller started for an early
member races the SB membership changes that later joins trigger.

Add three layers of recovery (the ansible bring-up settle pass that fixes this
at the source lives in the mulga repo):

- Per-host wedge watchdog (vpcd/ovn_controller_watchdog.go): not leader-gated
  since the wedge is per chassis. Polls SB connection-status; after a stale
  threshold issues sb-cluster-state-reset, throttled by a cooldown. Decision
  logic takes an injected clock for deterministic tests.
- Reconcile-loop escalation: ensureGatewayClaimed and ensureGuestPortDatapath
  escalate to one sb-cluster-state-reset after repeated recompute misses only
  when the SB is not connected — a recompute cannot clear a stale-SB wedge.
- Prober primitives SBConnectionState/ResetSBClusterState on GatewayClaimProber
  plus the GatewayClaimVerifier interface extension.

Also fix the awsgw OVN health probe, which passed a slashed `-t` socket path
that never resolves and so always reported ovn-controller not_running. Probe
connection-status instead: OVNControllerUp is now true only when connected, so
healthy nodes read ok and a real wedge correctly reads not_running.

.github/actions/e2e-analyze/analyze.go

@@ -1,7 +1,6 @@
 // Package main implements the e2e-analyze GitHub Action.
 //
-// Stage 1 of docs/development/improvements/e2e-go-failure-analysis.md:
-// cluster failures from go-junit-report's junit-*.xml by error signature,
+// Stage 1: cluster failures from go-junit-report's junit-*.xml by error signature,
 // pick the earliest non-cascade failure per suite as the likely root cause,
 // and render a human-readable report.
 //
@@ -463,8 +462,7 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 	return rep, nil
 }
 
-// Render writes the markdown report. Format is documented in
-// docs/development/improvements/e2e-go-failure-analysis.md (Stage 1).
+// Render writes the markdown report (Stage 1).
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)

.github/actions/e2e-analyze/bundle.go

@@ -1,7 +1,6 @@
 package main
 
-// Stage 2 of docs/development/improvements/e2e-go-failure-analysis.md:
-// for each failure surfaced by Stage 1, materialise a per-failure bundle
+// Stage 2: for each failure surfaced by Stage 1, materialise a per-failure bundle
 // directory containing the failure summary, a time-window slice of the
 // daemon journal, a testcase-window slice of the suite's stdout log, and
 // any per-test artifacts the harness already dumped at run-time.

.github/actions/e2e-analyze/slice.go

@@ -1,7 +1,6 @@
 package main
 
-// Stage 2 of docs/development/improvements/e2e-go-failure-analysis.md:
-// per-failure log slicers used by bundle.go to materialise time-window /
+// Stage 2: per-failure log slicers used by bundle.go to materialise time-window /
 // testcase-window log excerpts next to each named failure.
 
 import (

.github/workflows/e2e-ddil.yml

@@ -16,6 +16,10 @@ permissions:
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
   ddil:
@@ -200,6 +204,7 @@ jobs:
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
           quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
           ok "teardown complete"

.github/workflows/e2e-nightly.yml

@@ -17,6 +17,10 @@ permissions:
 env:
   GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
   UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
   matrix:
@@ -35,6 +39,7 @@ jobs:
           - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
           - { cell: 17, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
           - { cell: 18, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-single, extra: "reboot-single" }
+          - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }
 
     runs-on: [ self-hosted, "${{ matrix.runner }}" ]
     timeout-minutes: 35
@@ -121,7 +126,12 @@ jobs:
         env:
           MATRIX_NET: ${{ matrix.net }}
         run: |
-          if [ "$MATRIX_NET" = "dhcp" ]; then
+          if [ "$MATRIX_NET" = "nat" ]; then
+            # Routed-NAT mode with a static public pool: exercises both the
+            # Tier-1 transit datapath and the Tier-2 EIP ingress lane.
+            export SPX_EXTERNAL_MODE=nat
+            export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
+          elif [ "$MATRIX_NET" = "dhcp" ]; then
             unset SPX_EXTERNAL_POOL
             export SPX_EXTERNAL_SOURCE=dhcp
           else
@@ -154,6 +164,28 @@ jobs:
                if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
              done; exit \$FAIL"
 
+      - name: Run NAT Uplink E2E
+        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'nat-single'
+        working-directory: mulga/scripts/tofu-cluster
+        # Skips cleanly (exit 0) when the tested branch predates the natuplink
+        # suite so scheduled runs against dev stay green until the merge lands.
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
+             if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
+             echo '::group::suite natuplink'; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./natuplink.test -test.v -test.timeout=30m \
+               2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
+             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
+             if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
+
       - name: Run Multi Node E2E
         if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
         working-directory: mulga/scripts/tofu-cluster
@@ -312,6 +344,7 @@ jobs:
             TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
           fi
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
+          timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
 

.github/workflows/e2e-suites.yml

@@ -44,6 +44,10 @@ permissions:
 env:
   GIT_BRANCH: ${{ github.ref_name }}
   TEST_RUN: ${{ inputs.test_run || '' }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
   # ---------------------------------------------------------------------------
@@ -458,6 +462,7 @@ jobs:
           reap TERM; sleep 2; reap KILL
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
           quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
@@ -779,6 +784,7 @@ jobs:
           reap TERM; sleep 2; reap KILL
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
           quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"

.github/workflows/e2e.yml

@@ -31,6 +31,10 @@ env:
   GIT_BRANCH: ${{ github.ref_name }}
   SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam' }}
   TEST_RUN: ${{ inputs.test_run || '' }}
+  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
+  # empty string disables telemetry entirely.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
   # Shadow-mode suite selector. Computes which suites the diff *would* run and logs it to the job summary.
@@ -457,6 +461,7 @@ jobs:
           reap KILL
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
           quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
@@ -871,6 +876,7 @@ jobs:
           reap KILL
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
           quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"

.github/workflows/publish-eks-ami.yml

@@ -1,9 +1,11 @@
-name: Publish EKS node AMI
+name: Publish EKS + ECS node AMIs
 
-# Builds the eks-node system AMI (libguestfs, on the runner) and publishes the
-# compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
-# iso.mulgadc.com/system-ami/. Operators then install it on any cluster with:
+# Builds the eks-node and ecs-agent system AMIs (libguestfs, on the runner) and
+# publishes each compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket
+# that backs iso.mulgadc.com/system-ami/. Operators then install them on any
+# cluster with:
 #   spx admin images import --name spinifex-eks-node
+#   spx admin images import --name spinifex-ecs-node
 #
 # Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
 # action, not something every push should trigger.
@@ -25,7 +27,7 @@ concurrency:
 
 jobs:
   publish:
-    name: Build + publish eks-node AMI
+    name: Build + publish eks-node + ecs-agent AMIs
     runs-on: [ self-hosted, ci-single ]
 
     steps:
@@ -73,7 +75,7 @@ jobs:
             rm -rf /tmp/awscliv2.zip /tmp/aws-install
           fi
 
-      - name: Build + publish to R2
+      - name: Build + publish eks-node to R2
         working-directory: spinifex
         env:
           # Map the repo's R2 secrets onto the names publish-system-image.sh reads.
@@ -94,3 +96,22 @@ jobs:
           fi
           ./scripts/publish-system-image.sh \
             scripts/images/eks-node/manifest.conf "${FLAGS[@]}"
+
+      - name: Build + publish ecs-agent to R2
+        working-directory: spinifex
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
+          R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
+          R2_BUCKET: spinifex-iso
+          DRY_RUN: ${{ inputs.dry_run }}
+        run: |
+          set -euo pipefail
+          export LIBGUESTFS_BACKEND=direct
+          sudo chmod 0644 /boot/vmlinuz-* || true
+          FLAGS=(--build)
+          if [[ "${DRY_RUN}" == "true" ]]; then
+            FLAGS+=(--dry-run)
+          fi
+          ./scripts/publish-system-image.sh \
+            scripts/images/ecs-agent/manifest.conf "${FLAGS[@]}"

.github/workflows/release.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v6
         with:
-          go-version: "1.26.4"
+          go-version: "1.26.5"
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image

Makefile

@@ -218,9 +218,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.4 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.5 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.4.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.5.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi

build/systemd/spinifex-awsgw.service

@@ -5,6 +5,7 @@ After=spinifex-nats.service spinifex-daemon.service
 Wants=spinifex-nats.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-gw

build/systemd/spinifex-daemon.service

@@ -6,6 +6,7 @@ After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.servi
 Wants=openvswitch-ipsec.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-daemon

build/systemd/spinifex-nats.service

@@ -4,6 +4,7 @@ PartOf=spinifex.target
 Before=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-nats

build/systemd/spinifex-predastore.service

@@ -5,6 +5,10 @@ After=spinifex-nats.service
 Wants=spinifex-nats.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+# Guest block I/O arrives with no caller trace; sample those roots at 5% so
+# they don't drown request traces. Request-linked spans are always kept.
+Environment=MULGA_ROOT_TRACE_RATIO=0.05
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-storage

build/systemd/spinifex-qmp-collector.service

@@ -0,0 +1,56 @@
+[Unit]
+Description=Spinifex QMP Telemetry Collector (guest metrics)
+PartOf=spinifex.target
+After=spinifex-nats.service spinifex-daemon.service
+Wants=spinifex-nats.service
+
+[Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+Type=simple
+Slice=spinifex-system.slice
+# spinifex-daemon user: the collector reads the daemon's telemetry QMP
+# sockets in /run/spinifex and the daemon-owned QEMU /proc entries.
+User=spinifex-daemon
+Group=spinifex
+ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
+ExecStart=/usr/local/bin/spx service qmp-collector start
+Restart=on-failure
+RestartSec=5
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
+Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/
+# Same runtime dir as the daemon: telemetry sockets + metadata live there
+Environment=XDG_RUNTIME_DIR=/run/spinifex
+
+# RG-9: read-mostly telemetry tier — no capabilities, no device access
+ProtectSystem=full
+PrivateDevices=yes
+NoNewPrivileges=yes
+ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem
+ReadWritePaths=/var/lib/spinifex/spinifex /run/spinifex
+
+# RG-9: hardening baseline
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+# ProtectProc must stay default: hidepid=invisible hides the QEMU vCPU threads
+# (dumpable=0) this collector must read from /proc/<tid>/stat, even same-uid.
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=512
+
+[Install]
+WantedBy=spinifex.target

build/systemd/spinifex-ui.service

@@ -4,6 +4,7 @@ PartOf=spinifex.target
 After=spinifex-awsgw.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-ui

build/systemd/spinifex-viperblock.service

@@ -5,6 +5,7 @@ After=spinifex-nats.service spinifex-predastore.service
 Wants=spinifex-nats.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-viperblock

build/systemd/spinifex-vpcd.service

@@ -5,6 +5,7 @@ After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ope
 Wants=spinifex-nats.service ovn-controller.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-vpcd

build/systemd/spinifex.target

@@ -2,7 +2,7 @@
 Description=Spinifex Infrastructure Platform
 After=network-online.target
 Wants=network-online.target
-Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
+Wants=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
 
 [Install]
 WantedBy=multi-user.target

cmd/ecs-agent/controlplane.go

@@ -25,9 +25,10 @@ type controlPlane interface {
 	Register(id identity) error
 	// SubmitTaskState reports a task's RUNNING/STOPPED transition.
 	SubmitTaskState(st bus.TaskState) error
-	// PollAssignments drains this instance's assignment inbox, acking the taskIDs
-	// accepted on the previous poll. Returns the still-pending assignments.
-	PollAssignments(cluster, instance string, ack []string) ([]bus.Assign, error)
+	// PollAssignments drains this instance's assignment + stop inboxes, acking the
+	// assign taskIDs and stop taskIDs accepted on the previous poll. Returns the
+	// still-pending assignments and stop directives.
+	PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error)
 }
 
 // gatewayControlPlane implements controlPlane over the SigV4 ECS gateway client.
@@ -111,19 +112,24 @@ func (g *gatewayControlPlane) SubmitTaskState(st bus.TaskState) error {
 	return nil
 }
 
-func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ack []string) ([]bus.Assign, error) {
-	in := &handlers_ecs.PollAssignmentsInput{Cluster: cluster, ContainerInstance: instance, AckTaskIDs: ack}
+func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
+	in := &handlers_ecs.PollAssignmentsInput{
+		Cluster:           cluster,
+		ContainerInstance: instance,
+		AckTaskIDs:        ackAssigns,
+		AckStopIDs:        ackStops,
+	}
 	body, err := json.Marshal(in)
 	if err != nil {
-		return nil, fmt.Errorf("marshal poll: %w", err)
+		return nil, nil, fmt.Errorf("marshal poll: %w", err)
 	}
 	resp, err := g.client.Call("PollAssignments", body)
 	if err != nil {
-		return nil, fmt.Errorf("poll: %w", err)
+		return nil, nil, fmt.Errorf("poll: %w", err)
 	}
 	var out handlers_ecs.PollAssignmentsOutput
 	if err := json.Unmarshal(resp, &out); err != nil {
-		return nil, fmt.Errorf("decode poll: %w", err)
+		return nil, nil, fmt.Errorf("decode poll: %w", err)
 	}
-	return out.Assignments, nil
+	return out.Assignments, out.Stops, nil
 }

cmd/ecs-agent/credendpoint.go

@@ -192,6 +192,45 @@ func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Creden
 	return creds, roleARN, nil
 }
 
+// AssumeProvider returns a CredentialsProvider that mints credentials by assuming
+// roleARN over the gateway — the execution-role path for ECR image pulls. It
+// caches the assumed set and re-assumes within the refresh margin. session scopes
+// the STS session name.
+func (c *credEndpoint) AssumeProvider(roleARN, session string) credentials.CredentialsProvider {
+	return &assumeProvider{ep: c, roleARN: roleARN, session: session}
+}
+
+// assumeProvider adapts a credEndpoint's AssumeRole path to the agent's
+// CredentialsProvider interface, caching the assumed credentials.
+type assumeProvider struct {
+	ep      *credEndpoint
+	roleARN string
+	session string
+
+	mu     sync.Mutex
+	cached stsauth.Credentials
+}
+
+var _ credentials.CredentialsProvider = (*assumeProvider)(nil)
+
+func (p *assumeProvider) Retrieve(ctx context.Context) (credentials.Credentials, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if !credValid(p.cached, credRefreshMargin) {
+		creds, err := p.ep.assume(ctx, p.roleARN, p.session)
+		if err != nil {
+			return credentials.Credentials{}, err
+		}
+		p.cached = creds
+	}
+	return credentials.Credentials{
+		AccessKeyID:     p.cached.AccessKeyID,
+		SecretAccessKey: p.cached.SecretAccessKey,
+		SessionToken:    p.cached.SessionToken,
+		Expiration:      p.cached.Expiration,
+	}, nil
+}
+
 // assumeOverGateway mints credentials for roleARN by SigV4-signing AssumeRole
 // against the gateway with the agent's instance credentials.
 func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {

cmd/ecs-agent/registration_test.go

@@ -15,11 +15,13 @@ import (
 type fakeCP struct {
 	mu sync.Mutex
 
-	registers   int
-	states      []bus.TaskState
-	pollAcks    [][]string
-	pollReplies [][]bus.Assign
-	pollCalls   int
+	registers    int
+	states       []bus.TaskState
+	pollAcks     [][]string
+	pollStopAcks [][]string
+	pollReplies  [][]bus.Assign
+	stopReplies  [][]bus.StopDirective
+	pollCalls    int
 
 	registerErr bool
 	submitErr   bool
@@ -47,17 +49,21 @@ func (f *fakeCP) SubmitTaskState(st bus.TaskState) error {
 	return nil
 }
 
-func (f *fakeCP) PollAssignments(_, _ string, ack []string) ([]bus.Assign, error) {
+func (f *fakeCP) PollAssignments(_, _ string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	cp := append([]string(nil), ack...)
-	f.pollAcks = append(f.pollAcks, cp)
+	f.pollAcks = append(f.pollAcks, append([]string(nil), ackAssigns...))
+	f.pollStopAcks = append(f.pollStopAcks, append([]string(nil), ackStops...))
 	var out []bus.Assign
 	if f.pollCalls < len(f.pollReplies) {
 		out = f.pollReplies[f.pollCalls]
 	}
+	var stops []bus.StopDirective
+	if f.pollCalls < len(f.stopReplies) {
+		stops = f.stopReplies[f.pollCalls]
+	}
 	f.pollCalls++
-	return out, nil
+	return out, stops, nil
 }
 
 func (f *fakeCP) registerCount() int {
@@ -78,6 +84,12 @@ func (f *fakeCP) acks() [][]string {
 	return append([][]string(nil), f.pollAcks...)
 }
 
+func (f *fakeCP) stopAcks() [][]string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([][]string(nil), f.pollStopAcks...)
+}
+
 func testIdentity() identity {
 	return identity{
 		AccountID:    "123456789012",

cmd/ecs-agent/taskrun.go

@@ -22,31 +22,78 @@ func (a *Agent) pollAssignments(ctx context.Context, dispatched map[string]bool)
 	ticker := time.NewTicker(a.cfg.PollInterval)
 	defer ticker.Stop()
 
-	var ackNext []string
+	stopping := map[string]bool{}
+	var ackAssigns, ackStops []string
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-ticker.C:
-			assigns, err := a.cp.PollAssignments(a.id.ClusterName, a.id.InstanceID, ackNext)
+			assigns, stops, err := a.cp.PollAssignments(a.id.ClusterName, a.id.InstanceID, ackAssigns, ackStops)
 			if err != nil {
 				slog.Warn("ecs-agent: poll assignments failed", "err", err)
 				continue
 			}
-			ackNext = ackNext[:0]
+			// Dispatch stops first so a task stopped in the same poll it was assigned
+			// is never started.
+			ackStops = ackStops[:0]
+			for i := range stops {
+				sd := stops[i]
+				if !stopping[sd.TaskID] {
+					stopping[sd.TaskID] = true
+					slog.Info("ecs-agent: task stop requested", "task", sd.TaskID, "reason", sd.Reason)
+					go a.stopTask(ctx, sd)
+				}
+				ackStops = append(ackStops, sd.TaskID)
+			}
+			ackAssigns = ackAssigns[:0]
 			for i := range assigns {
 				as := assigns[i]
-				if !dispatched[as.TaskID] {
+				if !dispatched[as.TaskID] && !stopping[as.TaskID] {
 					dispatched[as.TaskID] = true
 					slog.Info("ecs-agent: task assigned", "task", as.TaskID, "containers", len(as.Containers))
 					go a.runTask(ctx, &as)
 				}
-				ackNext = append(ackNext, as.TaskID)
+				ackAssigns = append(ackAssigns, as.TaskID)
 			}
 		}
 	}
 }
 
+// stopTask reaps a task's containers on a control-plane stop directive: it lists
+// the runtime's containers, removes the ones labeled with this task (kill +
+// delete, releasing host ports/netns), then reports the task STOPPED with the
+// directive's reason. Idempotent — a task with no live containers still reports
+// STOPPED so the scheduler releases its capacity.
+func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) {
+	if a.runner == nil {
+		return
+	}
+	containers, err := a.runner.List(ctx)
+	if err != nil {
+		slog.Warn("ecs-agent: stop list failed", "task", sd.TaskID, "err", err)
+		return
+	}
+	statuses := make([]bus.ContainerStatus, 0)
+	for _, c := range containers {
+		if c.Labels[labelTaskID] != sd.TaskID {
+			continue
+		}
+		if rerr := a.runner.Remove(ctx, c.ID); rerr != nil {
+			slog.Warn("ecs-agent: stop remove failed", "task", sd.TaskID, "container", c.ID, "err", rerr)
+		}
+		statuses = append(statuses, bus.ContainerStatus{
+			Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
+		})
+	}
+	reason := sd.Reason
+	if reason == "" {
+		reason = "Task stopped"
+	}
+	a.reportTaskState(&bus.Assign{TaskID: sd.TaskID}, bus.TaskStatusStopped, reason, statuses)
+	slog.Info("ecs-agent: task reaped", "task", sd.TaskID, "containers", len(statuses))
+}
+
 // runTask pulls each container image, starts the containers, and reports task
 // state on the bus. A missing runtime or any per-container failure reports the
 // task STOPPED with a reason; success reports RUNNING and waits for exit.
@@ -65,10 +112,11 @@ func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
 	}
 
 	credID := a.registerTaskCreds(as)
+	resolver := a.pullResolver(as)
 
 	statuses := make([]bus.ContainerStatus, 0, len(as.Containers))
 	for _, c := range as.Containers {
-		if _, err := a.puller.Pull(ctx, ctrruntime.PullSpec{Ref: c.Image}, a.resolver); err != nil {
+		if _, err := a.puller.Pull(ctx, ctrruntime.PullSpec{Ref: c.Image}, resolver); err != nil {
 			slog.Error("ecs-agent: pull failed", "task", as.TaskID, "image", c.Image, "err", err)
 			a.teardownTaskNetns(as)
 			a.reportTaskState(as, bus.TaskStatusStopped, "image pull failed: "+err.Error(), statuses)
@@ -137,6 +185,18 @@ func (a *Agent) teardownTaskNetns(as *bus.Assign) {
 	}
 }
 
+// pullResolver returns the ECR resolver used for a task's image pulls. When the
+// assign carries an execution role, it authorizes pulls as that role (assumed
+// over the gateway); an empty role or a nil credential endpoint (unit tests)
+// falls back to the instance-role resolver, so a pull is never worse off.
+func (a *Agent) pullResolver(as *bus.Assign) ctrruntime.Resolver {
+	if as.ExecutionRoleARN == "" || a.cred == nil {
+		return a.resolver
+	}
+	prov := a.cred.AssumeProvider(as.ExecutionRoleARN, sessionName(as.TaskID))
+	return newLazyECRResolver(prov, a.cfg.Region, a.cfg.GatewayURL, a.cfg.GatewayCA)
+}
+
 // registerTaskCreds registers the task's credID -> role mapping with the
 // credential endpoint and returns the credID (empty when the task has no role).
 func (a *Agent) registerTaskCreds(as *bus.Assign) string {

cmd/ecs-agent/taskrun_test.go

@@ -172,6 +172,151 @@ func TestPollAssignments_DispatchesOnceAndAcks(t *testing.T) {
 	}
 }
 
+// stopTask reaps the task's labeled containers and reports STOPPED with the
+// directive reason and container status.
+func TestStopTask_ReapsLabeledContainersAndReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-web", Running: true, Labels: map[string]string{
+			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "web",
+		}},
+		{ID: "t-999-other", Running: true, Labels: map[string]string{
+			"mulga.ecs.taskID": "t-999", "mulga.ecs.containerName": "web",
+		}},
+	}}
+	a := newRunAgent(cp, rt)
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
+
+	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
+		t.Fatalf("want only t-001-web removed, got %+v", rt.Removed)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED report, got %+v", st)
+	}
+	if st[0].Reason != "bye" {
+		t.Errorf("want reason %q, got %q", "bye", st[0].Reason)
+	}
+	if len(st[0].Containers) != 1 || st[0].Containers[0].Status != bus.TaskStatusStopped {
+		t.Errorf("want one STOPPED container, got %+v", st[0].Containers)
+	}
+}
+
+// A stop for a task with no live containers still reports STOPPED so the
+// scheduler releases its capacity.
+func TestStopTask_NoContainersStillReportsStopped(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{}
+	a := newRunAgent(cp, rt)
+
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "gone"})
+
+	if len(rt.Removed) != 0 {
+		t.Errorf("nothing to remove, got %+v", rt.Removed)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED report, got %+v", st)
+	}
+}
+
+// A stop delivered by poll is dispatched once, acked in the stop-ack list, and
+// suppresses a same-poll assign for the same task.
+func TestPollAssignments_StopReapsAndSuppressesAssign(t *testing.T) {
+	as := testAssign()
+	cp := &fakeCP{
+		pollReplies: [][]bus.Assign{{*as}, {*as}, nil},
+		stopReplies: [][]bus.StopDirective{{{TaskID: as.TaskID, Reason: "bye"}}},
+	}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
+		{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
+	}}
+	a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, rt, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go a.pollAssignments(ctx, map[string]bool{})
+
+	deadline := time.After(time.Second)
+	for {
+		if stopped := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusStopped); stopped >= 1 {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatalf("stop never reaped; states=%+v", cp.taskStates())
+		case <-time.After(2 * time.Millisecond):
+		}
+	}
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+
+	// The assign for the stopping task must never have started a container.
+	if len(rt.Runs) != 0 {
+		t.Errorf("assign for a stopping task should not run, got %+v", rt.Runs)
+	}
+	if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 0 {
+		t.Errorf("stopping task should never report RUNNING, got %d", running)
+	}
+	acked := false
+	for _, ack := range cp.stopAcks() {
+		for _, id := range ack {
+			if id == as.TaskID {
+				acked = true
+			}
+		}
+	}
+	if !acked {
+		t.Errorf("stop %s was never acked; stopAcks=%v", as.TaskID, cp.stopAcks())
+	}
+}
+
+// stopTask with no runtime is a no-op: nothing to reap, no state reported.
+func TestStopTask_NilRunnerNoOp(t *testing.T) {
+	cp := &fakeCP{}
+	a := newAgent(config{}, testIdentity(), cp, nil, nil, nil)
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "x"})
+	if len(cp.taskStates()) != 0 {
+		t.Fatalf("nil runner should report no state, got %+v", cp.taskStates())
+	}
+}
+
+// A List failure aborts the reap without reporting a (false) STOPPED state.
+func TestStopTask_ListErrorNoReport(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{ListErr: errors.New("list boom")}
+	a := newRunAgent(cp, rt)
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "x"})
+	if len(cp.taskStates()) != 0 {
+		t.Fatalf("list error should report no state, got %+v", cp.taskStates())
+	}
+}
+
+// A directive with no reason reaps the matching container and reports STOPPED with
+// the default reason.
+func TestStopTask_EmptyReasonReapsWithDefault(t *testing.T) {
+	cp := &fakeCP{}
+	rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{{
+		ID: "t-001-web",
+		Labels: map[string]string{
+			"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "web",
+		},
+	}}}
+	a := newRunAgent(cp, rt)
+	a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001"})
+
+	if len(rt.Removed) != 1 || rt.Removed[0] != "t-001-web" {
+		t.Fatalf("want container t-001-web reaped, got %+v", rt.Removed)
+	}
+	st := cp.taskStates()
+	if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
+		t.Fatalf("want one STOPPED report, got %+v", st)
+	}
+	if st[0].Reason != "Task stopped" {
+		t.Errorf("want default reason %q, got %q", "Task stopped", st[0].Reason)
+	}
+}
+
 func countStatus(states []bus.TaskState, taskID, status string) int {
 	n := 0
 	for _, st := range states {
@@ -227,3 +372,32 @@ func TestContainerID(t *testing.T) {
 		t.Errorf("containerID = %q, want t-1-web", got)
 	}
 }
+
+// sentinelResolver is an identity-comparable Resolver for pullResolver tests.
+type sentinelResolver struct{}
+
+func (sentinelResolver) Authorize(context.Context, string) (string, string, string, error) {
+	return "", "", "", nil
+}
+
+// TestPullResolver covers siv-459: an assign with no execution role (or no
+// credential endpoint) uses the instance resolver; with both, a distinct
+// exec-role resolver is built.
+func TestPullResolver(t *testing.T) {
+	fake := sentinelResolver{}
+	a := newAgent(config{Region: "us-east-1"}, testIdentity(), &fakeCP{}, nil, nil, fake)
+
+	if got := a.pullResolver(&bus.Assign{TaskID: "t1"}); got != fake {
+		t.Fatalf("no execution role: want instance resolver fallback")
+	}
+	execAssign := &bus.Assign{TaskID: "t1", ExecutionRoleARN: "arn:aws:iam::1:role/e"}
+	if got := a.pullResolver(execAssign); got != fake {
+		t.Fatalf("nil cred endpoint: want instance resolver fallback")
+	}
+
+	a.cred = &credEndpoint{}
+	got := a.pullResolver(execAssign)
+	if got == nil || got == fake {
+		t.Fatalf("execution role + cred endpoint: want distinct exec-role resolver, got %v", got)
+	}
+}

cmd/eks-gateway-fetch/main.go

@@ -17,10 +17,14 @@
 // Usage:
 //
 //	eks-gateway-fetch -resource addons   # GET /clusters/{cluster}/internal-addons/{accountId}
+//	eks-gateway-fetch -resource recovery -instance-id i-… # GET /clusters/{cluster}/internal-recovery/{accountId}/{instanceId}
+//
+// The recovery resource returns a single epoch\taction\tsnapshot line the on-VM
+// k3s-recovery agent applies before k3s starts (etcd cluster-reset / wipe-rejoin).
 //
 // Flags default to environment variables seeded by cloud-init:
 // EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
-// EKS_ACCOUNT_ID, EKS_CLUSTER_NAME.
+// EKS_ACCOUNT_ID, EKS_CLUSTER_NAME, EKS_INSTANCE_ID.
 package main
 
 import (
@@ -68,6 +72,7 @@ func main() {
 		accountID   string
 		clusterName string
 		resource    string
+		instanceID  string
 	)
 
 	flag.StringVar(&gatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
@@ -77,7 +82,8 @@ func main() {
 	flag.StringVar(&region, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
 	flag.StringVar(&accountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
 	flag.StringVar(&clusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
-	flag.StringVar(&resource, "resource", "addons", "Resource to fetch: addons")
+	flag.StringVar(&instanceID, "instance-id", os.Getenv("EKS_INSTANCE_ID"), "This member's EC2 instance ID (recovery resource)")
+	flag.StringVar(&resource, "resource", "addons", "Resource to fetch: addons | recovery")
 	flag.Parse()
 
 	switch {
@@ -85,15 +91,29 @@ func main() {
 		fatal("--account-id is required (or set EKS_ACCOUNT_ID)")
 	case clusterName == "":
 		fatal("--cluster is required (or set EKS_CLUSTER_NAME)")
-	case resource != "addons":
-		fatal("--resource must be addons")
+	case resource != "addons" && resource != "recovery":
+		fatal("--resource must be addons or recovery")
+	case resource == "recovery" && instanceID == "":
+		fatal("--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)")
 	}
 
 	client, err := eksgw.New(gatewayURL, gatewayCA, accessKey, secretKey, region)
 	if err != nil {
 		fatal(fmt.Sprintf("build gateway client: %v", err))
 	}
-	path := "/clusters/" + clusterName + "/internal-addons/" + accountID
+
+	var (
+		path string
+		emit func(io.Writer, []byte) error
+	)
+	switch resource {
+	case "recovery":
+		path = "/clusters/" + clusterName + "/internal-recovery/" + accountID + "/" + instanceID
+		emit = emitRecoveryTSV
+	default:
+		path = "/clusters/" + clusterName + "/internal-addons/" + accountID
+		emit = emitAddonsTSV
+	}
 
 	var lastErr error
 	for attempt := 1; attempt <= maxAttempts; attempt++ {
@@ -106,7 +126,7 @@ func main() {
 			}
 			continue
 		}
-		if werr := emitAddonsTSV(os.Stdout, body); werr != nil {
+		if werr := emit(os.Stdout, body); werr != nil {
 			fatal(fmt.Sprintf("render response: %v", werr))
 		}
 		return
@@ -133,6 +153,36 @@ func emitAddonsTSV(out io.Writer, body []byte) error {
 	return w.Flush()
 }
 
+// recoveryDirective mirrors the gateway's RecoveryDirective wire shape; duplicated
+// here to keep this tiny VM binary free of the handler package's dependency graph.
+type recoveryDirective struct {
+	Epoch    int64  `json:"epoch"`
+	Action   string `json:"action"`
+	Snapshot string `json:"snapshot"`
+}
+
+type internalRecoveryResponse struct {
+	Directive recoveryDirective `json:"directive"`
+}
+
+// emitRecoveryTSV decodes the recovery directive and writes a single
+// epoch\taction\tsnapshot line the on-VM k3s-recovery agent consumes without a
+// JSON parser. An unset action defaults to "none" (steady state).
+func emitRecoveryTSV(out io.Writer, body []byte) error {
+	var resp internalRecoveryResponse
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return fmt.Errorf("unmarshal internal-recovery response: %w", err)
+	}
+	d := resp.Directive
+	if d.Action == "" {
+		d.Action = "none"
+	}
+	if _, err := fmt.Fprintf(out, "%d\t%s\t%s\n", d.Epoch, d.Action, d.Snapshot); err != nil {
+		return err
+	}
+	return nil
+}
+
 func fatal(msg string) {
 	slog.Error("eks-gateway-fetch: " + msg)
 	os.Exit(1)

cmd/eks-konnectivity-cert/main.go

@@ -0,0 +1,245 @@
+// Command eks-konnectivity-cert mints, then reuses, an ECDSA-P256 serving
+// certificate for the on-node konnectivity-server, signed by the K3s cluster CA.
+//
+// The konnectivity-agent (DaemonSet on workers) verifies the server it dials
+// using the in-pod ServiceAccount CA (/var/run/secrets/.../ca.crt), which in
+// K3s is the same server-ca that signs the apiserver. Signing the konnectivity
+// serving cert with that CA is therefore what lets the agent validate the
+// reverse-tunnel endpoint with no separate CA distribution.
+//
+// The eks-node AMI carries no openssl, so this is the Go equivalent: load the
+// K3s server CA cert+key, issue a leaf (ServerAuth) cert carrying the cluster's
+// konnectivity SANs (the NLB private-endpoint IP, public endpoint IP, NLB DNS),
+// and cache it under -dir so the OpenRC service is byte-stable across restarts.
+//
+// Usage:
+//
+//	eks-konnectivity-cert -dir /var/lib/spinifex-eks/konnectivity \
+//	    -ca-cert /var/lib/rancher/k3s/server/tls/server-ca.crt \
+//	    -ca-key  /var/lib/rancher/k3s/server/tls/server-ca.key \
+//	    -cn konnectivity-server -sans 10.32.100.4,203.0.113.9,eks-toc.example
+//
+// Writes <dir>/tls.crt and <dir>/tls.key; prints the two paths (tab-separated).
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"log/slog"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func main() {
+	dir := flag.String("dir", "", "directory to cache tls.crt/tls.key in")
+	caCert := flag.String("ca-cert", "", "PEM path of the signing (K3s server) CA cert")
+	caKey := flag.String("ca-key", "", "PEM path of the signing CA private key")
+	cn := flag.String("cn", "konnectivity-server", "certificate common name")
+	sansCSV := flag.String("sans", "", "comma-separated SANs (IP or DNS)")
+	flag.Parse()
+
+	if *dir == "" || *caCert == "" || *caKey == "" || *sansCSV == "" {
+		fatal("eks-konnectivity-cert: -dir, -ca-cert, -ca-key and -sans are required")
+	}
+	dns, ips := splitSANs(*sansCSV)
+	if len(dns) == 0 && len(ips) == 0 {
+		fatal("eks-konnectivity-cert: -sans lists no usable names")
+	}
+
+	certPath, keyPath, err := ensureCert(*dir, *caCert, *caKey, *cn, dns, ips)
+	if err != nil {
+		fatal("eks-konnectivity-cert: " + err.Error())
+	}
+	fmt.Printf("%s\t%s\n", certPath, keyPath)
+}
+
+// ensureCert returns the cached cert/key paths under dir, minting and persisting
+// a fresh CA-signed pair when none is present, readable, and still SAN-current.
+func ensureCert(dir, caCertPath, caKeyPath, cn string, dns []string, ips []net.IP) (string, string, error) {
+	certPath := filepath.Join(dir, "tls.crt")
+	keyPath := filepath.Join(dir, "tls.key")
+
+	if certCoversSANs(certPath, keyPath, dns, ips) {
+		return certPath, keyPath, nil
+	}
+
+	caCert, caKey, err := loadCA(caCertPath, caKeyPath)
+	if err != nil {
+		return "", "", err
+	}
+	certPEM, keyPEM, err := issueLeaf(caCert, caKey, cn, dns, ips)
+	if err != nil {
+		return "", "", err
+	}
+	if err = os.MkdirAll(dir, 0o700); err != nil {
+		return "", "", fmt.Errorf("mkdir %s: %w", dir, err)
+	}
+	if err = os.WriteFile(certPath, certPEM, 0o644); err != nil {
+		return "", "", fmt.Errorf("write %s: %w", certPath, err)
+	}
+	if err = os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
+		return "", "", fmt.Errorf("write %s: %w", keyPath, err)
+	}
+	return certPath, keyPath, nil
+}
+
+// certCoversSANs reports whether the cached cert exists, parses, has a key, and
+// already carries every requested SAN — so a cert minted before the endpoint
+// set changed is re-issued rather than served stale.
+func certCoversSANs(certPath, keyPath string, dns []string, ips []net.IP) bool {
+	c, err := os.ReadFile(certPath)
+	if err != nil {
+		return false
+	}
+	if k, kerr := os.ReadFile(keyPath); kerr != nil || len(k) == 0 {
+		return false
+	}
+	der := decodeFirst(c)
+	if der == nil {
+		return false
+	}
+	crt, err := x509.ParseCertificate(der)
+	if err != nil {
+		return false
+	}
+	have := make(map[string]struct{}, len(crt.DNSNames)+len(crt.IPAddresses))
+	for _, n := range crt.DNSNames {
+		have[n] = struct{}{}
+	}
+	for _, ip := range crt.IPAddresses {
+		have[ip.String()] = struct{}{}
+	}
+	for _, n := range dns {
+		if _, ok := have[n]; !ok {
+			return false
+		}
+	}
+	for _, ip := range ips {
+		if _, ok := have[ip.String()]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+// loadCA parses the PEM CA cert + EC/PKCS8 private key used to sign the leaf.
+func loadCA(certPath, keyPath string) (*x509.Certificate, *ecdsa.PrivateKey, error) {
+	certPEM, err := os.ReadFile(certPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("read ca cert %s: %w", certPath, err)
+	}
+	der := decodeFirst(certPEM)
+	if der == nil {
+		return nil, nil, fmt.Errorf("ca cert %s: no PEM block", certPath)
+	}
+	caCert, err := x509.ParseCertificate(der)
+	if err != nil {
+		return nil, nil, fmt.Errorf("parse ca cert: %w", err)
+	}
+
+	keyPEM, err := os.ReadFile(keyPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("read ca key %s: %w", keyPath, err)
+	}
+	keyDER := decodeFirst(keyPEM)
+	if keyDER == nil {
+		return nil, nil, fmt.Errorf("ca key %s: no PEM block", keyPath)
+	}
+	caKey, err := parseECKey(keyDER)
+	if err != nil {
+		return nil, nil, err
+	}
+	return caCert, caKey, nil
+}
+
+// parseECKey accepts the EC SEC1 or PKCS8 encodings K3s may write for the CA key.
+func parseECKey(der []byte) (*ecdsa.PrivateKey, error) {
+	if k, err := x509.ParseECPrivateKey(der); err == nil {
+		return k, nil
+	}
+	k8, err := x509.ParsePKCS8PrivateKey(der)
+	if err != nil {
+		return nil, fmt.Errorf("parse ca key (sec1 and pkcs8 failed): %w", err)
+	}
+	ec, ok := k8.(*ecdsa.PrivateKey)
+	if !ok {
+		return nil, fmt.Errorf("ca key is %T, want ecdsa", k8)
+	}
+	return ec, nil
+}
+
+// issueLeaf mints an ECDSA-P256 ServerAuth leaf for cn + SANs, signed by caCert/caKey.
+func issueLeaf(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, cn string, dns []string, ips []net.IP) (certPEM, keyPEM []byte, err error) {
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, nil, fmt.Errorf("generate key: %w", err)
+	}
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return nil, nil, fmt.Errorf("serial: %w", err)
+	}
+	now := time.Now().UTC()
+	tmpl := x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: cn},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		DNSNames:              dns,
+		IPAddresses:           ips,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, caCert, &priv.PublicKey, caKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("create cert: %w", err)
+	}
+	keyDER, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, nil, fmt.Errorf("marshal key: %w", err)
+	}
+	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
+	return certPEM, keyPEM, nil
+}
+
+// splitSANs partitions the CSV into DNS names and IPs (an entry that parses as an
+// IP goes to IPAddresses, else DNSNames), matching how SAN matching is typed.
+func splitSANs(s string) (dns []string, ips []net.IP) {
+	for p := range strings.SplitSeq(s, ",") {
+		p = strings.TrimSpace(p)
+		if p == "" {
+			continue
+		}
+		if ip := net.ParseIP(p); ip != nil {
+			ips = append(ips, ip)
+			continue
+		}
+		dns = append(dns, p)
+	}
+	return dns, ips
+}
+
+// decodeFirst returns the DER bytes of the first PEM block, or nil.
+func decodeFirst(pemBytes []byte) []byte {
+	blk, _ := pem.Decode(pemBytes)
+	if blk == nil {
+		return nil
+	}
+	return blk.Bytes
+}
+
+func fatal(msg string) {
+	slog.Error(msg)
+	os.Exit(1)
+}

cmd/eks-konnectivity-cert/main_test.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// writeTestCA mints a self-signed EC CA (cert + SEC1 key PEM) into dir, returning
+// the two paths — a stand-in for K3s' /var/lib/rancher/k3s/server/tls/server-ca.*.
+func writeTestCA(t *testing.T, dir string) (certPath, keyPath string) {
+	t.Helper()
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("gen ca key: %v", err)
+	}
+	tmpl := x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "k3s-server-ca"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().AddDate(10, 0, 0),
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
+	if err != nil {
+		t.Fatalf("create ca cert: %v", err)
+	}
+	keyDER, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		t.Fatalf("marshal ca key: %v", err)
+	}
+	certPath = filepath.Join(dir, "server-ca.crt")
+	keyPath = filepath.Join(dir, "server-ca.key")
+	if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
+		t.Fatalf("write ca cert: %v", err)
+	}
+	if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
+		t.Fatalf("write ca key: %v", err)
+	}
+	return certPath, keyPath
+}
+
+func leafFrom(t *testing.T, certPath string) *x509.Certificate {
+	t.Helper()
+	b, err := os.ReadFile(certPath)
+	if err != nil {
+		t.Fatalf("read leaf: %v", err)
+	}
+	crt, err := x509.ParseCertificate(decodeFirst(b))
+	if err != nil {
+		t.Fatalf("parse leaf: %v", err)
+	}
+	return crt
+}
+
+func TestEnsureCertSignsLeafWithSANsThatChainsToCA(t *testing.T) {
+	dir := t.TempDir()
+	caCert, caKey := writeTestCA(t, dir)
+
+	dns, ips := splitSANs("10.32.100.4,203.0.113.9,eks-toc.example")
+	certPath, keyPath, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips)
+	if err != nil {
+		t.Fatalf("ensureCert: %v", err)
+	}
+	if _, err := os.Stat(keyPath); err != nil {
+		t.Fatalf("key not written: %v", err)
+	}
+
+	leaf := leafFrom(t, certPath)
+	if leaf.IsCA {
+		t.Error("konnectivity serving cert must be a leaf, not a CA")
+	}
+	// The agent dials the NLB private-endpoint IP and verifies via the in-pod CA,
+	// so the IP must be a SAN or the tunnel TLS handshake fails.
+	if err := leaf.VerifyHostname("10.32.100.4"); err != nil {
+		t.Errorf("IP SAN 10.32.100.4 missing: %v", err)
+	}
+
+	// The leaf must chain to the K3s server CA — that CA is the in-pod ca.crt the
+	// agent trusts, so no separate CA distribution is needed.
+	caPEM, _ := os.ReadFile(caCert)
+	roots := x509.NewCertPool()
+	if !roots.AppendCertsFromPEM(caPEM) {
+		t.Fatal("append CA failed")
+	}
+	if _, err := leaf.Verify(x509.VerifyOptions{
+		Roots:     roots,
+		DNSName:   "eks-toc.example",
+		KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}); err != nil {
+		t.Errorf("leaf does not chain to the K3s server CA: %v", err)
+	}
+}
+
+func TestEnsureCertReusesThenReissuesOnSANChange(t *testing.T) {
+	dir := t.TempDir()
+	caCert, caKey := writeTestCA(t, dir)
+
+	dns, ips := splitSANs("10.32.100.4,eks-toc.example")
+	c1, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips)
+	if err != nil {
+		t.Fatalf("first ensureCert: %v", err)
+	}
+	first, _ := os.ReadFile(c1)
+
+	// Same SANs → cached pair reused byte-for-byte (a churning cert would bounce
+	// the konnectivity-server on every restart).
+	if _, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips); err != nil {
+		t.Fatalf("second ensureCert: %v", err)
+	}
+	if second, _ := os.ReadFile(c1); string(first) != string(second) {
+		t.Error("ensureCert re-minted instead of reusing the SAN-current cached cert")
+	}
+
+	// A new endpoint in the SAN set must force a re-issue.
+	dns2, ips2 := splitSANs("10.32.100.4,eks-toc.example,10.32.100.99")
+	if _, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns2, ips2); err != nil {
+		t.Fatalf("third ensureCert: %v", err)
+	}
+	if third, _ := os.ReadFile(c1); string(first) == string(third) {
+		t.Error("ensureCert served a stale cert that does not cover the new SAN")
+	}
+	if err := leafFrom(t, c1).VerifyHostname("10.32.100.99"); err != nil {
+		t.Errorf("re-issued cert missing new SAN: %v", err)
+	}
+}
+
+func TestSplitSANsPartitionsIPsAndDNS(t *testing.T) {
+	dns, ips := splitSANs(" 10.0.0.1 , host.example , , 2001:db8::1 ")
+	if len(dns) != 1 || dns[0] != "host.example" {
+		t.Errorf("dns = %v, want [host.example]", dns)
+	}
+	wantIPs := []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("2001:db8::1")}
+	if len(ips) != len(wantIPs) {
+		t.Fatalf("ips = %v, want %v", ips, wantIPs)
+	}
+	for i, ip := range ips {
+		if !ip.Equal(wantIPs[i]) {
+			t.Errorf("ips[%d] = %v, want %v", i, ip, wantIPs[i])
+		}
+	}
+}

cmd/eks-token-webhook/cert.go

@@ -18,14 +18,10 @@ import (
 	"github.com/mulgadc/spinifex/internal/tlsconfig"
 )
 
-// tlsCertificate is the webhook's loopback serving certificate.
-type tlsCertificate struct {
-	cert tls.Certificate
-}
-
-func (c tlsCertificate) serverTLSConfig() *tls.Config {
+// serverTLSConfig is the webhook's loopback serving TLS config.
+func serverTLSConfig(cert tls.Certificate) *tls.Config {
 	return &tls.Config{
-		Certificates:     []tls.Certificate{c.cert},
+		Certificates:     []tls.Certificate{cert},
 		MinVersion:       tls.VersionTLS13,
 		CurvePreferences: tlsconfig.Curves,
 	}
@@ -36,30 +32,30 @@ func (c tlsCertificate) serverTLSConfig() *tls.Config {
 // across restarts keeps the CA the apiserver loaded into its webhook kubeconfig
 // valid even if the webhook process is restarted. Returns the parsed cert and
 // the certificate PEM (for embedding as the kubeconfig CA bundle).
-func ensureServingCert(certPath, keyPath string) (tlsCertificate, []byte, error) {
+func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error) {
 	if certPEM, keyPEM, ok := readCertPair(certPath, keyPath); ok {
 		cert, err := tls.X509KeyPair(certPEM, keyPEM)
 		if err == nil {
-			return tlsCertificate{cert: cert}, certPEM, nil
+			return cert, certPEM, nil
 		}
 		// Fall through to regenerate on a corrupt/incompatible persisted pair.
 	}
 
 	certPEM, keyPEM, err := generateSelfSigned()
 	if err != nil {
-		return tlsCertificate{}, nil, err
+		return tls.Certificate{}, nil, err
 	}
 	if err := os.WriteFile(certPath, certPEM, 0o644); err != nil {
-		return tlsCertificate{}, nil, fmt.Errorf("write cert %s: %w", certPath, err)
+		return tls.Certificate{}, nil, fmt.Errorf("write cert %s: %w", certPath, err)
 	}
 	if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
-		return tlsCertificate{}, nil, fmt.Errorf("write key %s: %w", keyPath, err)
+		return tls.Certificate{}, nil, fmt.Errorf("write key %s: %w", keyPath, err)
 	}
 	cert, err := tls.X509KeyPair(certPEM, keyPEM)
 	if err != nil {
-		return tlsCertificate{}, nil, fmt.Errorf("parse minted keypair: %w", err)
+		return tls.Certificate{}, nil, fmt.Errorf("parse minted keypair: %w", err)
 	}
-	return tlsCertificate{cert: cert}, certPEM, nil
+	return cert, certPEM, nil
 }
 
 func readCertPair(certPath, keyPath string) (certPEM, keyPEM []byte, ok bool) {

cmd/eks-token-webhook/main.go

@@ -10,6 +10,9 @@
 // On startup it writes the apiserver webhook kubeconfig (with its self-signed
 // serving CA) so k3s can be pointed at it via
 // --authentication-token-webhook-config-file.
+//
+// SigV4 creds come from the AWS SDK chain (IMDS instance-role) when static
+// EKS_ACCESS_KEY/EKS_SECRET_KEY are absent, matching the sibling helpers.
 package main
 
 import (
@@ -56,11 +59,13 @@ func loadConfig() (config, error) {
 		keyPath:     envOr("EKS_WEBHOOK_KEY", "/etc/spinifex-eks/token-webhook.key"),
 		kubeconfig:  envOr("EKS_WEBHOOK_KUBECONFIG", "/etc/spinifex-eks/token-webhook.kubeconfig"),
 	}
+	// Static SigV4 creds are optional: when absent, eksgw.New signs with the
+	// AWS SDK chain (IMDS instance-role creds), the same path the CP VM's
+	// sibling helpers use. The CP VM launches on an instance profile, so
+	// buildK3sUserData omits EKS_ACCESS_KEY/EKS_SECRET_KEY by design.
 	switch {
 	case c.gatewayURL == "":
 		return c, errors.New("EKS_GATEWAY_URL not set")
-	case c.accessKey == "" || c.secretKey == "":
-		return c, errors.New("EKS_ACCESS_KEY / EKS_SECRET_KEY not set")
 	case c.accountID == "":
 		return c, errors.New("EKS_ACCOUNT_ID not set")
 	case c.clusterName == "":
@@ -125,7 +130,12 @@ func run(cfg config) error {
 	})
 	mux.HandleFunc("/authenticate", authr.handle)
 
-	server := newServer(cfg.addr, mux, tlsCert)
+	server := &http.Server{
+		Addr:              cfg.addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+		TLSConfig:         serverTLSConfig(tlsCert),
+	}
 	slog.Info("eks-token-webhook listening", "addr", cfg.addr, "cluster", cfg.clusterName)
 	if err := server.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
 		return fmt.Errorf("serve: %w", err)
@@ -214,14 +224,3 @@ func (a *authenticator) handle(w http.ResponseWriter, r *http.Request) {
 		slog.Error("encode TokenReview response", "err", err)
 	}
 }
-
-// newServer builds the loopback TLS server. Split out so tests can construct it
-// without binding a port.
-func newServer(addr string, handler http.Handler, cert tlsCertificate) *http.Server {
-	return &http.Server{
-		Addr:              addr,
-		Handler:           handler,
-		ReadHeaderTimeout: 5 * time.Second,
-		TLSConfig:         cert.serverTLSConfig(),
-	}
-}

cmd/eks-token-webhook/main_test.go

@@ -99,7 +99,7 @@ func TestEnsureServingCert_PersistsAndReuses(t *testing.T) {
 	c1, pem1, err := ensureServingCert(certPath, keyPath)
 	require.NoError(t, err)
 	require.NotEmpty(t, pem1)
-	require.NotNil(t, c1.serverTLSConfig())
+	require.NotNil(t, serverTLSConfig(c1))
 
 	// Second call reuses the persisted pair (same cert bytes).
 	_, pem2, err := ensureServingCert(certPath, keyPath)
@@ -121,3 +121,41 @@ func TestWriteAPIServerKubeconfig(t *testing.T) {
 	assert.Contains(t, body, base64.StdEncoding.EncodeToString(certPEM))
 	assert.Contains(t, body, "current-context: webhook")
 }
+
+// The CP VM launches on an instance profile, so cloud-init omits the static
+// EKS_ACCESS_KEY/EKS_SECRET_KEY and eksgw.New signs with IMDS creds. loadConfig
+// must accept absent static keys; gating on them aborts every IMDS-mode boot,
+// leaving the apiserver webhook kubeconfig unwritten.
+func TestLoadConfig_AcceptsAbsentStaticCreds(t *testing.T) {
+	setEnv := func(t *testing.T, kv map[string]string) {
+		t.Helper()
+		for _, k := range []string{
+			"EKS_GATEWAY_URL", "EKS_ACCESS_KEY", "EKS_SECRET_KEY",
+			"EKS_ACCOUNT_ID", "EKS_CLUSTER_NAME",
+		} {
+			t.Setenv(k, "")
+		}
+		for k, v := range kv {
+			t.Setenv(k, v)
+		}
+	}
+
+	// IMDS mode: required vars set, static creds absent → accepted.
+	setEnv(t, map[string]string{
+		"EKS_GATEWAY_URL":  "https://gw:9999",
+		"EKS_ACCOUNT_ID":   "000000000001",
+		"EKS_CLUSTER_NAME": "toc",
+	})
+	cfg, err := loadConfig()
+	require.NoError(t, err)
+	assert.Empty(t, cfg.accessKey)
+	assert.Empty(t, cfg.secretKey)
+
+	// Genuinely-required vars are still enforced.
+	setEnv(t, map[string]string{
+		"EKS_ACCOUNT_ID":   "000000000001",
+		"EKS_CLUSTER_NAME": "toc",
+	})
+	_, err = loadConfig()
+	require.Error(t, err)
+}

cmd/installer/autoinstall/autoinstall.go

@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package autoinstall reads headless install parameters from environment
 // variables exported by spinifex-init.sh from the kernel cmdline.
 // Parameters are set in the GRUB "Headless" menu entry in grub.cfg.

cmd/installer/branding/branding.go

@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package branding centralises Mulga/Spinifex visual identity constants used
 // by the installer TUI.
 //

cmd/installer/firstboot/firstboot.go

@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package firstboot writes the oneshot systemd service and configuration that
 // completes Spinifex provisioning on the first real boot after installation.
 package firstboot
@@ -91,6 +74,31 @@ func writeScript(root string, cfg Config) error {
 		setupOVN += fmt.Sprintf(" --encap-ip=%s", cfg.EncapIP)
 	}
 
+	// Pre-start OVS and OVN central so their databases are initialised before
+	// setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes
+	// longer than setup-ovn.sh's internal 15-second timeout allows. Starting them
+	// here and waiting until the NB DB is ready means setup-ovn.sh sees a live DB
+	// the moment it starts — no races, no timeout failures.
+	ovnPrestart := `systemctl start openvswitch-switch
+systemctl start ovn-central
+echo "Waiting for OVN NB DB to initialise..."
+for _i in $(seq 1 120); do
+    if ovn-nbctl --timeout=2 get-connection >/dev/null 2>&1; then
+        echo "OVN NB DB ready (${_i}s)"
+        break
+    fi
+    sleep 1
+done`
+
+	// When a provisioning controller owns formation (SkipFormation) it also owns
+	// the clustered OVN bring-up via setup-ovn.sh's RAFT flags. firstboot must
+	// not start a standalone ovn-central here: the single-node .db it would leave
+	// blocks ovn-ctl's create/join (which require a clean DB). Defer both.
+	if cfg.SkipFormation {
+		ovnPrestart = `echo "[firstboot] OVN bring-up deferred to provisioning controller"`
+		setupOVN = `echo "[firstboot] setup-ovn deferred to provisioning controller"`
+	}
+
 	callbackBlock := ""
 	if cfg.InstallCallback != "" {
 		callbackBlock = fmt.Sprintf(
@@ -123,21 +131,7 @@ fi
 # Set hostname
 hostnamectl set-hostname %s
 
-# Pre-start OVS and OVN central so their databases are initialised before
-# setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes
-# longer than setup-ovn.sh's internal 15-second timeout allows. Starting them
-# here and waiting until the NB DB is ready means setup-ovn.sh sees a live DB
-# the moment it starts — no races, no timeout failures.
-systemctl start openvswitch-switch
-systemctl start ovn-central
-echo "Waiting for OVN NB DB to initialise..."
-for _i in $(seq 1 120); do
-    if ovn-nbctl --timeout=2 get-connection >/dev/null 2>&1; then
-        echo "OVN NB DB ready (${_i}s)"
-        break
-    fi
-    sleep 1
-done
+%s
 
 # Create default nameservers if DHCP server returns blank
 printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" > /etc/resolvconf/resolv.conf.d/base
@@ -229,7 +223,7 @@ if grep -q 'external_mode.*pool' /etc/spinifex/spinifex.toml 2>/dev/null; then
         echo "[firstboot] warning: br-ext not up after 30s — external networking may be delayed"
     fi
 fi
-`, cfg.Hostname, setupOVN, clusterCmd, callbackBlock)
+`, cfg.Hostname, ovnPrestart, setupOVN, clusterCmd, callbackBlock)
 
 	path := filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh")
 	return os.WriteFile(path, []byte(script), 0o755)

cmd/installer/firstboot/firstboot_test.go

@@ -66,6 +66,54 @@ func TestWriteScriptEmbedsCurlWhenCallbackSet(t *testing.T) {
 	}
 }
 
+func TestWriteScriptRunsOVNWhenFormationOwned(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node", ClusterRole: "init"}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	content := readScript(t, root)
+
+	if !strings.Contains(content, "setup-ovn.sh --management") {
+		t.Error("script should run setup-ovn --management when firstboot owns formation")
+	}
+	if !strings.Contains(content, "systemctl start ovn-central") {
+		t.Error("script should pre-start ovn-central when firstboot owns formation")
+	}
+}
+
+func TestWriteScriptDefersOVNWhenSkipFormation(t *testing.T) {
+	root := t.TempDir()
+	makeRootDirs(t, root)
+
+	cfg := Config{Hostname: "test-node", ClusterRole: "init", SkipFormation: true}
+	if err := Write(root, cfg); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	content := readScript(t, root)
+
+	if strings.Contains(content, "setup-ovn.sh --management") {
+		t.Error("script must not run setup-ovn --management when a controller owns OVN")
+	}
+	if strings.Contains(content, "systemctl start ovn-central") {
+		t.Error("script must not pre-start ovn-central when a controller owns OVN")
+	}
+	if !strings.Contains(content, "setup-ovn deferred") {
+		t.Error("script should note setup-ovn is deferred under SkipFormation")
+	}
+}
+
+func readScript(t *testing.T, root string) string {
+	t.Helper()
+	b, err := os.ReadFile(filepath.Join(root, "usr/local/bin/spinifex-firstboot.sh"))
+	if err != nil {
+		t.Fatalf("read script: %v", err)
+	}
+	return string(b)
+}
+
 func TestWriteScriptCallbackAfterDoneMarker(t *testing.T) {
 	root := t.TempDir()
 	makeRootDirs(t, root)

cmd/installer/install/config.go

@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package install performs the disk installation steps after the UI has
 // collected configuration.
 package install

cmd/installer/install/install.go

@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 package install
 
 import (

cmd/installer/main.go

@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package main
 
 import (

cmd/installer/systemd/systemd.go

@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program. If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package systemd writes systemd unit files and drop-ins into an installed
 // system root during the Spinifex ISO installation process.
 package systemd

cmd/installer/ui/ui.go

@@ -1,20 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
 // Package ui presents the interactive installer TUI using bubbletea and lipgloss.
 package ui
 

cmd/lb-agent/main.go

@@ -49,12 +49,13 @@ func main() {
 		slog.Error("--gateway is required (or set LB_GATEWAY_URL)")
 		os.Exit(1)
 	}
-	if accessKey == "" || secretKey == "" {
-		slog.Error("--access-key and --secret-key are required (or set LB_ACCESS_KEY / LB_SECRET_KEY)")
-		os.Exit(1)
+	// Empty keys select IMDS instance-role credentials (the SDK default chain
+	// falls back to the in-VM IMDS endpoint); static keys sign as-is.
+	credSource := "imds"
+	if accessKey != "" && secretKey != "" {
+		credSource = "static"
 	}
-
-	slog.Info("Starting LB agent", "lbId", lbID, "gateway", gatewayURL)
+	slog.Info("Starting LB agent", "lbId", lbID, "gateway", gatewayURL, "credSource", credSource)
 
 	agent, err := lbagent.New(lbID, gatewayURL, accessKey, secretKey, region)
 	if err != nil {

cmd/spinifex/LICENSE

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

cmd/spinifex/cmd/admin.go

@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd
 
 import (
@@ -50,6 +34,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/gpu"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/mulgadc/spinifex/spinifex/network/host"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/viperblock/viperblock"
@@ -194,6 +179,18 @@ unless --force is passed. Prompts for confirmation unless --yes is passed.`,
 	Run: runimagesRemoveCmd,
 }
 
+var imagesPromoteCmd = &cobra.Command{
+	Use:   "promote",
+	Short: "Promote an account-owned AMI to a system image",
+	Long: `Rewrite an account-owned AMI's owner to the system alias so it becomes
+visible to all accounts via DescribeImages, matching the behaviour of AMIs
+imported via 'spx admin images import'.
+
+No data is copied — only the config.json owner field is updated. The change
+takes effect immediately. Prompts for confirmation unless --yes is passed.`,
+	Run: runimagesPromoteCmd,
+}
+
 var accountCmd = &cobra.Command{
 	Use:   "account",
 	Short: "Manage Spinifex accounts",
@@ -285,6 +282,7 @@ func init() {
 	imagesCmd.AddCommand(imagesImportCmd)
 	imagesCmd.AddCommand(imagesListCmd)
 	imagesCmd.AddCommand(imagesRemoveCmd)
+	imagesCmd.AddCommand(imagesPromoteCmd)
 
 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
@@ -328,7 +326,7 @@ func init() {
 	adminInitCmd.Flags().StringSlice("services", nil, "Services this node runs (default: all). Valid: nats,predastore,viperblock,daemon,awsgw,ui")
 
 	// External networking flags
-	adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (default when WAN detected) or '' (disabled)")
+	adminInitCmd.Flags().String("external-mode", "", "External network mode: 'pool' (default when WAN detected), 'nat' (routed; non-bridgeable uplinks; add --external-pool or --external-source=dhcp for public IPs), or '' (disabled)")
 	adminInitCmd.Flags().String("external-iface", "", "WAN NIC for br-external (auto-detected from default route)")
 	adminInitCmd.Flags().String("external-source", "", "Pool IP source: 'dhcp' (default when no --external-pool) or 'static' (uses --external-pool range)")
 	adminInitCmd.Flags().String("external-bind-bridge", "", "Linux bridge for upstream DHCP DORA (default 'br-wan' when --external-source=dhcp)")
@@ -380,6 +378,12 @@ func init() {
 	if err := imagesRemoveCmd.MarkFlagRequired("image-id"); err != nil {
 		panic(err)
 	}
+
+	imagesPromoteCmd.Flags().String("image-id", "", "AMI ID to promote to system image (required)")
+	imagesPromoteCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
+	if err := imagesPromoteCmd.MarkFlagRequired("image-id"); err != nil {
+		panic(err)
+	}
 }
 
 func runimagesImportCmd(cmd *cobra.Command, args []string) {
@@ -873,6 +877,67 @@ func printDependents(w io.Writer, d admin.Dependents) {
 	}
 }
 
+func runimagesPromoteCmd(cmd *cobra.Command, args []string) {
+	imageID, _ := cmd.Flags().GetString("image-id")
+	yes, _ := cmd.Flags().GetBool("yes")
+
+	cfgFile, _ := cmd.Flags().GetString("config")
+	if cfgFile == "" {
+		cfgFile = DefaultConfigFile()
+	}
+
+	appConfig, err := config.LoadConfig(cfgFile)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Error loading config file:", err)
+		os.Exit(1)
+	}
+
+	node := appConfig.Nodes[appConfig.Node]
+	store := objectstore.NewS3ObjectStoreFromConfig(
+		node.Predastore.Host,
+		node.Predastore.Region,
+		node.Predastore.AccessKey,
+		node.Predastore.SecretKey,
+	)
+	bucket := node.Predastore.Bucket
+
+	// Read current metadata for the confirmation prompt.
+	meta, err := admin.GetAMIMetadata(store, bucket, imageID)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Failed to inspect AMI:", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("About to promote AMI to system image:")
+	fmt.Println()
+	fmt.Printf("  Image ID:       %s\n", imageID)
+	fmt.Printf("  Name:           %s\n", meta.Name)
+	fmt.Printf("  Current owner:  %s\n", meta.ImageOwnerAlias)
+	fmt.Printf("  New owner:      %s\n", admin.SystemOwnerAlias)
+	if !meta.CreationDate.IsZero() {
+		fmt.Printf("  Created:        %s\n", meta.CreationDate.UTC().Format("2006-01-02T15:04:05Z"))
+	}
+	fmt.Println()
+	fmt.Println("After promotion this AMI will be visible to all accounts.")
+
+	if !yes {
+		fmt.Print("Type 'yes' to proceed: ")
+		reader := bufio.NewReader(os.Stdin)
+		answer, _ := reader.ReadString('\n')
+		if strings.TrimSpace(answer) != "yes" {
+			fmt.Println("Aborted.")
+			return
+		}
+	}
+
+	if _, err := admin.PromoteSystemImage(store, bucket, admin.PromoteImageOpts{ImageID: imageID}); err != nil {
+		fmt.Fprintln(os.Stderr, "Promote failed:", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("✅ Promoted %s to system image (owner: %s).\n", imageID, admin.SystemOwnerAlias)
+}
+
 // List remote images available
 func runimagesListCmd(cmd *cobra.Command, args []string) {
 	//fmt.Println(availableImages)
@@ -1026,6 +1091,13 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 				// --external-pool is omitted the validator below will error with
 				// a SuggestPoolRange hint.
 				if externalMode == "" && !cmd.Flags().Changed("external-mode") {
+					if isNonBridgeableUplink(detected.WAN.Name) {
+						fmt.Fprintf(os.Stderr, "\n❌ Detected WAN interface %s cannot be bridged (WiFi/cellular/PPP).\n", detected.WAN.Name)
+						fmt.Fprintf(os.Stderr, "   Use routed NAT mode instead (outbound-only VM networking):\n")
+						fmt.Fprintf(os.Stderr, "     ./scripts/setup-ovn.sh --management --nat-uplink\n")
+						fmt.Fprintf(os.Stderr, "     spx admin init --external-mode=nat\n")
+						os.Exit(1)
+					}
 					externalMode = "pool"
 				}
 			}
@@ -1033,60 +1105,60 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}
 
 	// Validate external networking flags
-	if externalMode != "" && externalMode != "pool" {
-		fmt.Fprintf(os.Stderr, "❌ Error: --external-mode must be 'pool' or empty, got: %s\n", externalMode)
+	if externalMode != "" && externalMode != "pool" && externalMode != "nat" {
+		fmt.Fprintf(os.Stderr, "❌ Error: --external-mode must be 'pool', 'nat', or empty, got: %s\n", externalMode)
 		os.Exit(1)
 	}
-	if externalMode == "pool" {
-		// Default source: dhcp when no --external-pool, else static.
-		if externalSource == "" {
-			if externalPool == "" {
-				externalSource = "dhcp"
-			} else {
-				externalSource = "static"
-			}
-		}
-		switch externalSource {
-		case "dhcp":
-			if externalPool != "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool not allowed with --external-source=dhcp (addresses come from upstream DHCP server)\n")
-				os.Exit(1)
-			}
-			if externalBindBridge == "" {
-				externalBindBridge = "br-wan"
-			}
-		case "static":
-			if externalBindBridge != "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-bind-bridge only valid with --external-source=dhcp\n")
-				os.Exit(1)
-			}
-			if externalPool == "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool is required with --external-source=static (e.g., 192.168.1.150-192.168.1.250)\n")
-				if detectedNet != nil && detectedNet.WAN != nil {
-					sugStart, sugEnd := admin.SuggestPoolRange(detectedNet.WAN)
-					fmt.Fprintf(os.Stderr, "   Suggested: --external-pool=%s-%s\n", sugStart, sugEnd)
-				}
-				os.Exit(1)
-			}
-			if externalGateway == "" {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is required with --external-source=static\n")
-				os.Exit(1)
-			}
-			parts := strings.SplitN(externalPool, "-", 2)
-			if len(parts) != 2 || net.ParseIP(parts[0]) == nil || net.ParseIP(parts[1]) == nil {
-				fmt.Fprintf(os.Stderr, "❌ Error: --external-pool must be start-end IPs (e.g., 192.168.1.150-192.168.1.250), got: %s\n", externalPool)
-				os.Exit(1)
-			}
-			poolStart, poolEnd = parts[0], parts[1]
-		default:
-			fmt.Fprintf(os.Stderr, "❌ Error: --external-source must be 'static' or 'dhcp', got: %s\n", externalSource)
+	// A public pool alongside nat's transit pool restores EIP / public-subnet
+	// parity where the operator has spare LAN IPs; without these flags nat
+	// stays Tier-1-only (host-jumpbox access, no public IPs).
+	natPublicPool := externalMode == "nat" && (externalPool != "" || externalSource != "")
+	// natPublicGateway keeps the upstream gateway for the public pool before
+	// the transit segment claims externalGateway below.
+	natPublicGateway := externalGateway
+	if externalMode == "nat" {
+		if nodes >= 2 {
+			fmt.Fprintf(os.Stderr, "❌ Error: --external-mode=nat is single-node only (v1); use --nodes=1\n")
 			os.Exit(1)
 		}
+		if !natPublicPool && (externalBindBridge != "" || gatewayIP != "") {
+			fmt.Fprintf(os.Stderr, "❌ Error: --external-bind-bridge/--gateway-ip require --external-pool or --external-source in --external-mode=nat\n")
+			os.Exit(1)
+		}
+		if natPublicPool {
+			// DHCP DORA in nat mode binds the uplink interface itself — there
+			// is no br-wan (nothing is bridged in routed mode).
+			src, start, end, bb, err := resolvePublicPoolFlags(externalSource, externalPool, externalBindBridge, natPublicGateway, externalIface)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+				os.Exit(1)
+			}
+			externalSource, poolStart, poolEnd, externalBindBridge = src, start, end, bb
+		}
+		// The transit segment is fixed: the host veth owns the gateway IP and
+		// masquerades the /24.
+		externalGateway = host.NATTransitGatewayIP
+	}
+	if externalMode == "pool" {
+		src, start, end, bb, err := resolvePublicPoolFlags(externalSource, externalPool, externalBindBridge, externalGateway, "br-wan")
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+			if strings.Contains(err.Error(), "--external-pool is required") && detectedNet != nil && detectedNet.WAN != nil {
+				sugStart, sugEnd := admin.SuggestPoolRange(detectedNet.WAN)
+				fmt.Fprintf(os.Stderr, "   Suggested: --external-pool=%s-%s\n", sugStart, sugEnd)
+			}
+			os.Exit(1)
+		}
+		externalSource, poolStart, poolEnd, externalBindBridge = src, start, end, bb
 	}
 	if externalGateway != "" && net.ParseIP(externalGateway) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is not a valid IP: %s\n", externalGateway)
 		os.Exit(1)
 	}
+	if natPublicGateway != "" && net.ParseIP(natPublicGateway) == nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is not a valid IP: %s\n", natPublicGateway)
+		os.Exit(1)
+	}
 	if gatewayIP != "" && net.ParseIP(gatewayIP) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: --gateway-ip is not a valid IP: %s\n", gatewayIP)
 		os.Exit(1)
@@ -1101,6 +1173,35 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		}
 	}
 
+	// Assemble the external pool blocks rendered into spinifex.toml.
+	var externalPools []admin.PoolData
+	switch externalMode {
+	case "nat":
+		externalPools = append(externalPools, admin.PoolData{
+			Name: host.NATTransitPoolName, Gateway: host.NATTransitGatewayIP,
+			PrefixLen: 24, DNSServers: dnsServers,
+		})
+		if natPublicPool {
+			// WiFi/WWAN uplinks drop frames with foreign source MACs, so DHCP
+			// leases must go out with the interface's own MAC.
+			dhcpMAC := ""
+			if externalSource == "dhcp" && isNonBridgeableUplink(externalBindBridge) {
+				dhcpMAC = "interface"
+			}
+			externalPools = append(externalPools, admin.PoolData{
+				Name: "wan", Source: externalSource, BindBridge: externalBindBridge,
+				DHCPMAC: dhcpMAC, Start: poolStart, End: poolEnd, Gateway: natPublicGateway,
+				GatewayIP: gatewayIP, PrefixLen: externalPrefixLen, DNSServers: dnsServers,
+			})
+		}
+	case "pool":
+		externalPools = append(externalPools, admin.PoolData{
+			Name: "wan", Source: externalSource, BindBridge: externalBindBridge,
+			Start: poolStart, End: poolEnd, Gateway: externalGateway,
+			GatewayIP: gatewayIP, PrefixLen: externalPrefixLen, DNSServers: dnsServers,
+		})
+	}
+
 	// Validate IP address format
 	if net.ParseIP(bindIP) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP address for --bind: %s\n", bindIP)
@@ -1386,17 +1487,10 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		OVNNBAddr: "tcp:127.0.0.1:6641",
 		OVNSBAddr: "tcp:127.0.0.1:6642",
 
-		ExternalMode:   externalMode,
-		ExternalIface:  externalIface,
-		PoolName:       "wan",
-		PoolSource:     externalSource,
-		PoolBindBridge: externalBindBridge,
-		PoolStart:      poolStart,
-		PoolEnd:        poolEnd,
-		PoolGateway:    externalGateway,
-		PoolGatewayIP:  gatewayIP,
-		PoolPrefixLen:  externalPrefixLen,
-		PoolDNSServers: dnsServers,
+		ExternalMode:  externalMode,
+		ExternalIface: externalIface,
+		BridgeMode:    bridgeModeFor(externalMode),
+		Pools:         externalPools,
 
 		OperatorEmail:       email,
 		BootstrapAccountId:  admin.DefaultAccountID(),
@@ -1413,7 +1507,20 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}
 
 	// Print external networking summary
-	if externalMode != "" {
+	if externalMode == "nat" {
+		if natPublicPool {
+			fmt.Printf("\n📡 External networking: nat (routed) with public pool — EIPs enabled\n")
+			if externalSource == "static" {
+				fmt.Printf("  Public pool:   %s - %s (source: static)\n", poolStart, poolEnd)
+			} else {
+				fmt.Printf("  Public pool:   dhcp via %s\n", externalBindBridge)
+			}
+		} else {
+			fmt.Printf("\n📡 External networking: nat (routed, outbound-only — no public IPs/EIPs)\n")
+		}
+		fmt.Printf("  Transit:       %s via %s (host masquerades out any uplink)\n", host.NATTransitCIDR, host.NATTransitHostEnd)
+		fmt.Printf("  Host setup:    ./scripts/setup-ovn.sh --nat-uplink (run before starting services)\n")
+	} else if externalMode != "" {
 		fmt.Printf("\n📡 External networking: %s\n", externalMode)
 		fmt.Printf("  WAN interface: %s\n", externalIface)
 		switch externalSource {
@@ -1616,6 +1723,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	allNodes := fs.Nodes()
 	clusterRoutes := formation.BuildClusterRoutes(allNodes)
 	predastoreNodes := formation.BuildPredastoreNodes(allNodes)
+	ovnNBAddr, ovnSBAddr := formation.BuildOVNDBAddrs(allNodes)
 
 	fmt.Println("\n📝 Creating configuration files...")
 
@@ -1677,9 +1785,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 		EncryptionKeyFile: viperblockKeyPath,
 
-		// Init node runs ovn-central locally
-		OVNNBAddr: "tcp:127.0.0.1:6641",
-		OVNSBAddr: "tcp:127.0.0.1:6642",
+		// Multi-endpoint OVN NB/SB list across the RAFT quorum; the init node's
+		// own address leads, the rest provide failover.
+		OVNNBAddr: ovnNBAddr,
+		OVNSBAddr: ovnSBAddr,
 	}
 
 	if networkConfig != nil {
@@ -1764,13 +1873,6 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Extract leader IP for OVN NB/SB DB address (strip port from host:port)
-	leaderIP, _, err := net.SplitHostPort(leaderHost)
-	if err != nil {
-		// leaderHost might be an IP without port
-		leaderIP = leaderHost
-	}
-
 	// Validate IP address format
 	if net.ParseIP(bindIP) == nil {
 		fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP address for --bind: %s\n", bindIP)
@@ -2073,6 +2175,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	// Build cluster topology from formation data
 	clusterRoutes := formation.BuildClusterRoutes(statusResp.Nodes)
 	predastoreNodes := formation.BuildPredastoreNodes(statusResp.Nodes)
+	ovnNBAddr, ovnSBAddr := formation.BuildOVNDBAddrs(statusResp.Nodes)
 
 	fmt.Println("📝 Creating configuration files...")
 
@@ -2139,9 +2242,10 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 		EncryptionKeyFile: viperblockKeyPath,
 
-		// Joining nodes connect to the init node's OVN NB/SB DB
-		OVNNBAddr: fmt.Sprintf("tcp:%s:6641", leaderIP),
-		OVNSBAddr: fmt.Sprintf("tcp:%s:6642", leaderIP),
+		// Multi-endpoint OVN NB/SB list across the RAFT quorum so the client
+		// fails over instead of pinning to a single init node.
+		OVNNBAddr: ovnNBAddr,
+		OVNSBAddr: ovnSBAddr,
 	}
 
 	if statusResp.NetworkConfig != nil {
@@ -2410,7 +2514,7 @@ func runCertRenew(cmd *cobra.Command, _ []string) {
 	serverCertPath := filepath.Join(configDir, "server.pem")
 	serverKeyPath := filepath.Join(configDir, "server.key")
 
-	if err := admin.GenerateSignedCertWithDNS(serverCertPath, serverKeyPath, caCertPath, caKeyPath, extraIPs, extraDNS); err != nil {
+	if err := admin.GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, extraIPs, extraDNS); err != nil {
 		fmt.Fprintf(os.Stderr, "Error regenerating server certificate: %v\n", err)
 		os.Exit(1)
 	}
@@ -2486,15 +2590,19 @@ func finalizeNodeSetup(dataDir, certPath, adminAccessKey, adminSecretKey, region
 func applyNetworkConfig(settings *admin.ConfigSettings, nc *formation.NetworkConfig) {
 	settings.IPSecEnabled = nc.IPSecEnabled
 	settings.ExternalMode = nc.ExternalMode
-	settings.PoolName = nc.PoolName
-	settings.PoolSource = nc.PoolSource
-	settings.PoolBindBridge = nc.PoolBindBridge
-	settings.PoolStart = nc.PoolStart
-	settings.PoolEnd = nc.PoolEnd
-	settings.PoolGateway = nc.PoolGateway
-	settings.PoolGatewayIP = nc.PoolGatewayIP
-	settings.PoolPrefixLen = nc.PoolPrefixLen
-	settings.PoolDNSServers = nc.PoolDNSServers
+	if nc.ExternalMode != "" {
+		settings.Pools = []admin.PoolData{{
+			Name:       nc.PoolName,
+			Source:     nc.PoolSource,
+			BindBridge: nc.PoolBindBridge,
+			Start:      nc.PoolStart,
+			End:        nc.PoolEnd,
+			Gateway:    nc.PoolGateway,
+			GatewayIP:  nc.PoolGatewayIP,
+			PrefixLen:  nc.PoolPrefixLen,
+			DNSServers: nc.PoolDNSServers,
+		}}
+	}
 
 	settings.BootstrapAccountId = nc.BootstrapAccountId
 	settings.BootstrapVpcId = nc.BootstrapVpcId
@@ -2652,6 +2760,66 @@ func writeBootstrapFilesWithAdmin(configDir, bootstrapDir string, masterKey []by
 	return handlers_iam.SaveBootstrapData(filepath.Join(bootstrapDir, "bootstrap.json"), bd)
 }
 
+// isNonBridgeableUplink reports whether the interface name indicates an uplink
+// that cannot be enslaved to an L2 bridge: WiFi (wl*), cellular (ww*), PPP.
+func isNonBridgeableUplink(name string) bool {
+	return strings.HasPrefix(name, "wl") || strings.HasPrefix(name, "ww") || strings.HasPrefix(name, "ppp")
+}
+
+// resolvePublicPoolFlags validates the public-pool flag set shared by pool
+// mode and nat-with-public-pool. Returns the resolved source, static range
+// start/end, and bind bridge. Source defaults to dhcp when no range is given;
+// defaultBindBridge fills --external-bind-bridge for dhcp (br-wan in pool
+// mode, the uplink interface in nat mode).
+func resolvePublicPoolFlags(source, poolRange, bindBridge, gateway, defaultBindBridge string) (resolvedSource, start, end, resolvedBindBridge string, err error) {
+	if source == "" {
+		if poolRange == "" {
+			source = "dhcp"
+		} else {
+			source = "static"
+		}
+	}
+	switch source {
+	case "dhcp":
+		if poolRange != "" {
+			return "", "", "", "", fmt.Errorf("--external-pool not allowed with --external-source=dhcp (addresses come from upstream DHCP server)")
+		}
+		if bindBridge == "" {
+			if defaultBindBridge == "" {
+				return "", "", "", "", fmt.Errorf("--external-bind-bridge is required with --external-source=dhcp (no uplink interface detected)")
+			}
+			bindBridge = defaultBindBridge
+		}
+	case "static":
+		if bindBridge != "" {
+			return "", "", "", "", fmt.Errorf("--external-bind-bridge only valid with --external-source=dhcp")
+		}
+		if poolRange == "" {
+			return "", "", "", "", fmt.Errorf("--external-pool is required with --external-source=static (e.g., 192.168.1.150-192.168.1.250)")
+		}
+		if gateway == "" {
+			return "", "", "", "", fmt.Errorf("--external-gateway is required with --external-source=static")
+		}
+		parts := strings.SplitN(poolRange, "-", 2)
+		if len(parts) != 2 || net.ParseIP(parts[0]) == nil || net.ParseIP(parts[1]) == nil {
+			return "", "", "", "", fmt.Errorf("--external-pool must be start-end IPs (e.g., 192.168.1.150-192.168.1.250), got: %s", poolRange)
+		}
+		start, end = parts[0], parts[1]
+	default:
+		return "", "", "", "", fmt.Errorf("--external-source must be 'static' or 'dhcp', got: %s", source)
+	}
+	return source, start, end, bindBridge, nil
+}
+
+// bridgeModeFor returns the vpcd bridge_mode admin init persists for the
+// external mode: only nat mode is pinned; bridged modes stay auto-detected.
+func bridgeModeFor(externalMode string) string {
+	if externalMode == "nat" {
+		return "nat"
+	}
+	return ""
+}
+
 // detectDNSServers auto-detects DNS servers from the host for the specified
 // interface. Uses resolvectl (systemd-resolved) first, then falls back to
 // /etc/resolv.conf. Returns up to 3 servers. Falls back to public DNS if none found.

cmd/spinifex/cmd/admin_nat_test.go

@@ -0,0 +1,198 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSpinifexTomlTemplate_NATMode(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:      "node1",
+		Az:        "ap-southeast-2a",
+		Port:      "4432",
+		Region:    "ap-southeast-2",
+		BindIP:    "10.11.12.1",
+		AccessKey: "AKIATEST",
+		SecretKey: "SECRET",
+		AccountID: "123456789012",
+		NatsToken: "token",
+		ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "nat",
+		BridgeMode:   "nat",
+		Pools: []admin.PoolData{{
+			Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24,
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `external_mode = "nat"`)
+	assert.Contains(t, content, `bridge_mode = "nat"`)
+	assert.Contains(t, content, `name        = "nat-transit"`)
+	assert.Contains(t, content, `gateway     = "100.127.0.1"`)
+	assert.NotContains(t, content, "range_start", "nat mode has no public IP range")
+}
+
+func TestSpinifexTomlTemplate_PoolModeOmitsBridgeMode(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:      "node1",
+		Az:        "ap-southeast-2a",
+		Port:      "4432",
+		Region:    "ap-southeast-2",
+		BindIP:    "10.11.12.1",
+		AccessKey: "AKIATEST",
+		SecretKey: "SECRET",
+		AccountID: "123456789012",
+		NatsToken: "token",
+		ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode:  "pool",
+		ExternalIface: "enp0s3",
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "static",
+			Start: "192.168.1.150", End: "192.168.1.250",
+			Gateway: "192.168.1.1", PrefixLen: 24,
+		}},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	assert.NotContains(t, string(data), "bridge_mode", "bridged modes keep runtime auto-detect")
+}
+
+func TestIsNonBridgeableUplink(t *testing.T) {
+	for _, name := range []string{"wlan0", "wlp3s0", "wlx001122334455", "wwan0", "wwp0s20f0u6", "ppp0"} {
+		assert.True(t, isNonBridgeableUplink(name), name)
+	}
+	for _, name := range []string{"eth0", "enp0s3", "eno1", "br-wan", "veth-wan-br"} {
+		assert.False(t, isNonBridgeableUplink(name), name)
+	}
+}
+
+func TestBridgeModeFor(t *testing.T) {
+	assert.Equal(t, "nat", bridgeModeFor("nat"))
+	assert.Equal(t, "", bridgeModeFor("pool"))
+	assert.Equal(t, "", bridgeModeFor(""))
+}
+
+func TestSpinifexTomlTemplate_NATModeWithPublicPool(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	settings := admin.ConfigSettings{
+		Node:      "node1",
+		Az:        "ap-southeast-2a",
+		Port:      "4432",
+		Region:    "ap-southeast-2",
+		BindIP:    "10.11.12.1",
+		AccessKey: "AKIATEST",
+		SecretKey: "SECRET",
+		AccountID: "123456789012",
+		NatsToken: "token",
+		ConfigDir: dir,
+		OVNNBAddr: "tcp:127.0.0.1:6641",
+		OVNSBAddr: "tcp:127.0.0.1:6642",
+
+		ExternalMode: "nat",
+		BridgeMode:   "nat",
+		Pools: []admin.PoolData{
+			{Name: "nat-transit", Gateway: "100.127.0.1", PrefixLen: 24},
+			{
+				Name: "wan", Source: "dhcp", BindBridge: "wlan0",
+				DHCPMAC: "interface", PrefixLen: 24,
+			},
+		},
+	}
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	content := string(data)
+	assert.Contains(t, content, `name        = "nat-transit"`)
+	assert.Contains(t, content, `name        = "wan"`)
+	assert.Contains(t, content, `source      = "dhcp"`)
+	assert.Contains(t, content, `bind_bridge = "wlan0"`)
+	assert.Contains(t, content, `dhcp_mac    = "interface"`)
+	transitIdx := strings.Index(content, `name        = "nat-transit"`)
+	wanIdx := strings.Index(content, `name        = "wan"`)
+	assert.Less(t, transitIdx, wanIdx, "transit pool must render first (IGW allocator picks it)")
+}
+
+func TestResolvePublicPoolFlags(t *testing.T) {
+	tests := []struct {
+		name              string
+		source, poolRange string
+		bindBridge, gw    string
+		defaultBridge     string
+		wantSource        string
+		wantStart         string
+		wantEnd           string
+		wantBridge        string
+		wantErr           string
+	}{
+		{
+			name: "static pool range", source: "", poolRange: "192.168.1.150-192.168.1.250", gw: "192.168.1.1",
+			wantSource: "static", wantStart: "192.168.1.150", wantEnd: "192.168.1.250",
+		},
+		{
+			name: "dhcp defaults bind bridge", source: "dhcp", defaultBridge: "wlan0",
+			wantSource: "dhcp", wantBridge: "wlan0",
+		},
+		{
+			name: "dhcp explicit bind bridge", source: "dhcp", bindBridge: "br-wan", defaultBridge: "wlan0",
+			wantSource: "dhcp", wantBridge: "br-wan",
+		},
+		{
+			name: "dhcp rejects pool range", source: "dhcp", poolRange: "192.168.1.150-192.168.1.250",
+			wantErr: "--external-pool not allowed",
+		},
+		{
+			name: "static requires gateway", poolRange: "192.168.1.150-192.168.1.250",
+			wantErr: "--external-gateway is required",
+		},
+		{
+			name: "static rejects bind bridge", poolRange: "192.168.1.150-192.168.1.250", gw: "192.168.1.1", bindBridge: "br-wan",
+			wantErr: "--external-bind-bridge only valid",
+		},
+		{
+			name: "static requires range", source: "static", gw: "192.168.1.1",
+			wantErr: "--external-pool is required",
+		},
+		{
+			name: "bad range", poolRange: "not-a-range", gw: "192.168.1.1",
+			wantErr: "start-end IPs",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			src, start, end, bb, err := resolvePublicPoolFlags(tt.source, tt.poolRange, tt.bindBridge, tt.gw, tt.defaultBridge)
+			if tt.wantErr != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, tt.wantSource, src)
+			assert.Equal(t, tt.wantStart, start)
+			assert.Equal(t, tt.wantEnd, end)
+			assert.Equal(t, tt.wantBridge, bb)
+		})
+	}
+}

cmd/spinifex/cmd/admin_test.go

@@ -382,20 +382,19 @@ func TestSpinifexTomlTemplate_ExternalPoolDHCPSource(t *testing.T) {
 	dir := t.TempDir()
 	path := filepath.Join(dir, "spinifex.toml")
 	settings := admin.ConfigSettings{
-		Node:           "node1",
-		Az:             "ap-southeast-2a",
-		Port:           "4432",
-		Region:         "ap-southeast-2",
-		BindIP:         "10.11.12.1",
-		ConfigDir:      dir,
-		OVNNBAddr:      "tcp:127.0.0.1:6641",
-		OVNSBAddr:      "tcp:127.0.0.1:6642",
-		ExternalMode:   "pool",
-		ExternalIface:  "eth0",
-		PoolName:       "wan",
-		PoolSource:     "dhcp",
-		PoolBindBridge: "br-wan",
-		PoolPrefixLen:  24,
+		Node:          "node1",
+		Az:            "ap-southeast-2a",
+		Port:          "4432",
+		Region:        "ap-southeast-2",
+		BindIP:        "10.11.12.1",
+		ConfigDir:     dir,
+		OVNNBAddr:     "tcp:127.0.0.1:6641",
+		OVNSBAddr:     "tcp:127.0.0.1:6642",
+		ExternalMode:  "pool",
+		ExternalIface: "eth0",
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "dhcp", BindBridge: "br-wan", PrefixLen: 24,
+		}},
 	}
 	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
 
@@ -424,12 +423,11 @@ func TestSpinifexTomlTemplate_ExternalPoolStaticSource(t *testing.T) {
 		OVNSBAddr:     "tcp:127.0.0.1:6642",
 		ExternalMode:  "pool",
 		ExternalIface: "eth0",
-		PoolName:      "wan",
-		PoolSource:    "static",
-		PoolStart:     "192.168.1.150",
-		PoolEnd:       "192.168.1.250",
-		PoolGateway:   "192.168.1.1",
-		PoolPrefixLen: 24,
+		Pools: []admin.PoolData{{
+			Name: "wan", Source: "static",
+			Start: "192.168.1.150", End: "192.168.1.250",
+			Gateway: "192.168.1.1", PrefixLen: 24,
+		}},
 	}
 	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
 
@@ -454,8 +452,9 @@ func TestApplyNetworkConfig_PropagatesPoolBindBridge(t *testing.T) {
 		PoolPrefixLen:  24,
 	}
 	applyNetworkConfig(settings, nc)
-	assert.Equal(t, "dhcp", settings.PoolSource)
-	assert.Equal(t, "br-wan", settings.PoolBindBridge)
+	require.Len(t, settings.Pools, 1)
+	assert.Equal(t, "dhcp", settings.Pools[0].Source)
+	assert.Equal(t, "br-wan", settings.Pools[0].BindBridge)
 }
 
 func renderSingleNodePredastore(t *testing.T, settings admin.ConfigSettings) string {

cmd/spinifex/cmd/get.go

@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd
 
 import (

cmd/spinifex/cmd/root.go

@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd
 
 import (

cmd/spinifex/cmd/service.go

@@ -1,32 +1,20 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"log/slog"
 	"os"
 	"strings"
 
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/network/external"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"github.com/mulgadc/spinifex/spinifex/service"
 	"github.com/mulgadc/spinifex/spinifex/services/awsgw"
 	"github.com/mulgadc/spinifex/spinifex/services/nats"
 	"github.com/mulgadc/spinifex/spinifex/services/predastore"
+	"github.com/mulgadc/spinifex/spinifex/services/qmpcollector"
 	"github.com/mulgadc/spinifex/spinifex/services/spinifexui"
 	"github.com/mulgadc/spinifex/spinifex/services/viperblockd"
 	"github.com/mulgadc/spinifex/spinifex/vpcd"
@@ -39,6 +27,28 @@ var serviceCmd = &cobra.Command{
 	Short: "Manage Spinifex services",
 }
 
+// initTelemetry installs the JSON slog default (trace-stamping) and the OTel
+// providers for a service process. The returned func flushes exporters and
+// must be deferred; with no OTLP endpoint configured both are no-ops.
+func initTelemetry(serviceName string, debug bool) func() {
+	level := slog.LevelInfo
+	if debug {
+		level = slog.LevelDebug
+	}
+	otelsetup.SetDefaultJSONLogger(level)
+
+	shutdown, err := otelsetup.Init(context.Background(), serviceName)
+	if err != nil {
+		slog.Warn("otel init", "service", serviceName, "error", err)
+		return func() {}
+	}
+	return func() {
+		if err := shutdown(context.Background()); err != nil {
+			slog.Warn("otel shutdown", "service", serviceName, "error", err)
+		}
+	}
+}
+
 var predastoreCmd = &cobra.Command{
 	Use:   "predastore",
 	Short: "Manage the predastore service",
@@ -126,6 +136,8 @@ var predastoreStartCmd = &cobra.Command{
 		pprofEnabled := viper.GetBool("pprof")
 		pprofOutput := viper.GetString("pprof-output")
 
+		defer initTelemetry("predastore", debug)()
+
 		service, err := service.New("predastore", &predastore.Config{
 			Port:       port,
 			Host:       host,
@@ -288,6 +300,8 @@ var viperblockStartCmd = &cobra.Command{
 			encryptionKeyFile = envKey
 		}
 
+		defer initTelemetry("viperblockd", false)()
+
 		service, err := service.New("viperblock", &viperblockd.Config{
 			NatsHost:          nodeConfig.NATS.Host,
 			NatsToken:         nodeConfig.NATS.ACL.Token,
@@ -366,6 +380,8 @@ var natsStartCmd = &cobra.Command{
 
 		cfgFile := viper.GetString("config")
 
+		defer initTelemetry("nats", debug)()
+
 		service, err := service.New("nats", &nats.Config{
 			ConfigFile: cfgFile,
 			Port:       port,
@@ -459,6 +475,8 @@ var spinifexStartCmd = &cobra.Command{
 		// Apply changes back to cluster config
 		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig
 
+		defer initTelemetry("spinifex-daemon", false)()
+
 		svc, err := service.New("spinifex", clusterConfig)
 
 		if err != nil {
@@ -564,6 +582,8 @@ var awsgwStartCmd = &cobra.Command{
 		// Apply changes back to cluster config
 		clusterConfig.Nodes[clusterConfig.Node] = nodeConfig
 
+		defer initTelemetry("awsgw", viper.GetBool("debug"))()
+
 		awsgw.SetBuildInfo(Version, Commit)
 		service, err := service.New("awsgw", clusterConfig)
 
@@ -621,6 +641,8 @@ var spinifexUIStartCmd = &cobra.Command{
 		tlsCert := viper.GetString("spinifex-ui-tls-cert")
 		tlsKey := viper.GetString("spinifex-ui-tls-key")
 
+		defer initTelemetry("spinifex-ui", false)()
+
 		svc, err := service.New("spinifex-ui", &spinifexui.Config{
 			Port:    port,
 			Host:    host,
@@ -730,12 +752,13 @@ var vpcdStartCmd = &cobra.Command{
 		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
 
 		// Map cluster-wide external pools to vpcd config
-		var extPools []vpcd.ExternalPoolConfig
+		var extPools []external.ExternalPoolConfig
 		for _, p := range clusterConfig.Network.ExternalPools {
-			extPools = append(extPools, vpcd.ExternalPoolConfig{
+			extPools = append(extPools, external.ExternalPoolConfig{
 				Name:            p.Name,
 				Source:          p.Source,
 				BindBridge:      p.BindBridge,
+				DHCPMAC:         p.DHCPMAC,
 				RangeStart:      p.RangeStart,
 				RangeEnd:        p.RangeEnd,
 				Gateway:         p.Gateway,
@@ -767,6 +790,8 @@ var vpcdStartCmd = &cobra.Command{
 			nodeConfig.BaseDir = baseDir
 		}
 
+		defer initTelemetry("vpcd", false)()
+
 		svc, err := service.New("vpcd", &vpcd.Config{
 			NatsHost:          nodeConfig.NATS.Host,
 			NatsToken:         nodeConfig.NATS.ACL.Token,
@@ -781,6 +806,7 @@ var vpcdStartCmd = &cobra.Command{
 			ExternalInterface: nodeConfig.VPCD.ExternalInterface,
 			BridgeMode:        nodeConfig.VPCD.BridgeMode,
 			AZ:                nodeConfig.AZ,
+			NATExemptCIDRs:    clusterConfig.Network.NATExemptCIDRs,
 		})
 		if err != nil {
 			fmt.Println("Error starting vpcd service:", err)
@@ -823,6 +849,77 @@ var vpcdStatusCmd = &cobra.Command{
 	},
 }
 
+var qmpCollectorCmd = &cobra.Command{
+	Use:   "qmp-collector",
+	Short: "Manage the qmp-collector (guest metrics) service",
+}
+
+var qmpCollectorStartCmd = &cobra.Command{
+	Use:   "start",
+	Short: "Start the qmp-collector service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("Starting qmp-collector service...")
+
+		cfgFile := viper.GetString("config")
+		if cfgFile == "" {
+			fmt.Println("Config file is not set")
+			return
+		}
+		clusterConfig, err := config.LoadConfig(cfgFile)
+		if err != nil {
+			fmt.Println("Error loading config file:", err)
+			return
+		}
+		nodeConfig := clusterConfig.Nodes[clusterConfig.Node]
+
+		defer initTelemetry("qmp-collector", false)()
+
+		svc, err := service.New("qmp-collector", &qmpcollector.Config{
+			NatsHost:   nodeConfig.NATS.Host,
+			NatsToken:  nodeConfig.NATS.ACL.Token,
+			NatsCACert: nodeConfig.NATS.CACert,
+			BaseDir:    nodeConfig.BaseDir,
+			NodeName:   clusterConfig.Node,
+		})
+		if err != nil {
+			fmt.Println("Error starting qmp-collector service:", err)
+			return
+		}
+		if _, err = svc.Start(); err != nil {
+			fmt.Println("Error starting qmp-collector service:", err)
+			os.Exit(1)
+		}
+		fmt.Println("qmp-collector service started")
+	},
+}
+
+var qmpCollectorStopCmd = &cobra.Command{
+	Use:   "stop",
+	Short: "Stop the qmp-collector service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("Stopping qmp-collector service...")
+
+		svc, err := service.New("qmp-collector", &qmpcollector.Config{})
+		if err != nil {
+			fmt.Println("Error stopping qmp-collector service:", err)
+			return
+		}
+		if err = svc.Stop(); err != nil {
+			fmt.Println("Error stopping qmp-collector service:", err)
+			os.Exit(1)
+		}
+		fmt.Println("qmp-collector service stopped")
+	},
+}
+
+var qmpCollectorStatusCmd = &cobra.Command{
+	Use:   "status",
+	Short: "Get status of the qmp-collector service",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("qmp-collector service status: ...")
+	},
+}
+
 func init() {
 	viper.SetEnvPrefix("SPINIFEX") // Prefix for environment variables
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
@@ -1022,4 +1119,11 @@ func init() {
 	vpcdCmd.AddCommand(vpcdStartCmd)
 	vpcdCmd.AddCommand(vpcdStopCmd)
 	vpcdCmd.AddCommand(vpcdStatusCmd)
+
+	// qmp-collector
+	serviceCmd.AddCommand(qmpCollectorCmd)
+
+	qmpCollectorCmd.AddCommand(qmpCollectorStartCmd)
+	qmpCollectorCmd.AddCommand(qmpCollectorStopCmd)
+	qmpCollectorCmd.AddCommand(qmpCollectorStatusCmd)
 }

cmd/spinifex/cmd/telemetry_test.go

@@ -0,0 +1,24 @@
+package cmd
+
+import "testing"
+
+func TestInitTelemetryNoEndpoint(t *testing.T) {
+	t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
+
+	tests := []struct {
+		name  string
+		debug bool
+	}{
+		{"info level", false},
+		{"debug level", true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			flush := initTelemetry("test-svc", tt.debug)
+			if flush == nil {
+				t.Fatal("initTelemetry returned nil flush func")
+			}
+			flush()
+		})
+	}
+}

cmd/spinifex/cmd/templates/spinifex.toml

@@ -16,28 +16,33 @@ internal_suffix = "spinifex.internal"
 ipsec_enabled = {{.IPSecEnabled}}
 {{- if .ExternalMode}}
 external_mode = "{{.ExternalMode}}"
+{{- range .Pools}}
 
 [[network.external_pools]]
-name        = "{{.PoolName}}"
-{{- if .PoolSource}}
-source      = "{{.PoolSource}}"
+name        = "{{.Name}}"
+{{- if .Source}}
+source      = "{{.Source}}"
 {{- end}}
-{{- if .PoolBindBridge}}
-bind_bridge = "{{.PoolBindBridge}}"
+{{- if .BindBridge}}
+bind_bridge = "{{.BindBridge}}"
 {{- end}}
-{{- if .PoolStart}}
-range_start = "{{.PoolStart}}"
-range_end   = "{{.PoolEnd}}"
+{{- if .DHCPMAC}}
+dhcp_mac    = "{{.DHCPMAC}}"
 {{- end}}
-{{- if .PoolGateway}}
-gateway     = "{{.PoolGateway}}"
+{{- if .Start}}
+range_start = "{{.Start}}"
+range_end   = "{{.End}}"
 {{- end}}
-{{- if .PoolGatewayIP}}
-gateway_ip  = "{{.PoolGatewayIP}}"
+{{- if .Gateway}}
+gateway     = "{{.Gateway}}"
+{{- end}}
+{{- if .GatewayIP}}
+gateway_ip  = "{{.GatewayIP}}"
+{{- end}}
+prefix_len  = {{if .PrefixLen}}{{.PrefixLen}}{{else}}24{{end}}
+{{- if .DNSServers}}
+dns_servers = [{{range $i, $s := .DNSServers}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
 {{- end}}
-prefix_len  = {{if .PoolPrefixLen}}{{.PoolPrefixLen}}{{else}}24{{end}}
-{{- if .PoolDNSServers}}
-dns_servers = [{{range $i, $s := .PoolDNSServers}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
 {{- end}}
 {{- end}}
 {{- if .BootstrapVpcId}}
@@ -107,6 +112,9 @@ ovn_sb_addr = "{{.OVNSBAddr}}"
 {{- if .ExternalIface}}
 external_interface = "{{.ExternalIface}}"
 {{- end}}
+{{- if .BridgeMode}}
+bridge_mode = "{{.BridgeMode}}"
+{{- end}}
 
 [nodes.{{.Node}}.predastore]
 host = "{{.BindIP}}:8443"

cmd/spinifex/cmd/top.go

@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package cmd
 
 import (

cmd/spinifex/main.go

@@ -1,19 +1,3 @@
-/*
-Copyright © 2026 Mulga Defense Corporation
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
 package main
 
 import (

docs/COMMANDS.md

@@ -31,6 +31,9 @@ Service lifecycle commands for starting, stopping, and checking status of all Sp
 | `spx service vpcd start` | — | Loads cluster config → starts VPC daemon (subscribes to `vpc.*` NATS events, translates to OVN logical switches/ports/routers) |
 | `spx service vpcd stop` | — | Stops the vpcd service |
 | `spx service vpcd status` | — | Reports vpcd service status |
+| `spx service qmp-collector start` | — | Starts the guest-metrics collector (polls per-VM telemetry QMP sockets + tap counters, publishes CloudWatch-shaped series to NATS `metrics.ec2.*`) |
+| `spx service qmp-collector stop` | — | Stops the qmp-collector service |
+| `spx service qmp-collector status` | — | Reports qmp-collector service status |
 
 ### Cluster Inspection
 
@@ -51,7 +54,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force` | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` |
+| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; add `--external-pool start-end --external-gateway <ip>` or `--external-source=dhcp [--external-bind-bridge <iface>]` for a public pool with full EIP support), `--no-external` | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` |
 | `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations |
 
 ### Version
@@ -93,8 +96,69 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 |---------|-------|-------------|
 | `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify` | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. On every import a best-effort `virt-customize` step bakes the deployment CA (`<data-root>/config/ca.pem`) into the image's trust store before the block copy, so in-guest SDK-over-TLS calls to Spinifex endpoints trust the gateway from first boot; an image libguestfs cannot customize is imported as-is (CA-free) and logs a skip. Because the CA is fixed into the image, rotating the cluster CA requires re-importing affected images. |
 | `spx admin images list` | — | Lists available OS images that can be imported or downloaded |
+| `spx admin images promote` | `--image-id` (required), `--yes` | Reads `ami-<id>/config.json`, validates the AMI is account-owned, then rewrites `ImageOwnerAlias` to `"system"` in-place. No block data is copied. The change takes effect immediately — the AMI becomes visible to all accounts via `DescribeImages`. Prompts for confirmation (skipped with `--yes`). Already-system AMIs are refused. |
 | `spx admin images remove` | `--image-id` (required), `--force`, `--yes` | Loads `ami-<id>/config.json`, walks transitive dependents — copied snapshots whose `VolumeID == imageID`, volumes whose `SnapshotID` references the internal `snap-ami-<id>` or any derived snap, and account AMIs created via `CopyImage` whose `SnapshotID` is a derived snap — then prompts (skipped with `--yes`) before deleting `ami-<id>/config.json` (the DescribeImages barrier) followed by the rest of `ami-<id>/` and `snap-ami-<id>/`. Account-owned AMIs are refused with a hint pointing at `aws ec2 deregister-image` + `aws ec2 delete-snapshot`. `--force` bypasses the dependency, ownership and config-corrupt checks for salvage of orphaned blocks. |
 
+#### Image integrity verification (CMMC SI.L1-3.14.2)
+
+Catalog imports (`spx admin images import --name <name>`) verify the image
+against the catalog-declared SHA-256/SHA-512 digest before extraction. The sums
+file is fetched from the catalog `Checksum` URL over HTTPS only (cross-scheme
+redirects refused), and verification runs on both fresh downloads and cache
+hits so a poisoned cache is caught on the next import.
+
+On mismatch the import exits non-zero, the cached file is left on disk for
+inspection, and the printed guidance is `spx admin images import --name <name>
+--force` to re-download.
+
+`--file` imports skip verification by design: operator-supplied media is
+outside Spinifex's trust boundary and the operator is responsible for
+integrity (e.g. `sha256sum` against a trusted upstream digest before import).
+The skip is recorded as an INFO `slog` event with `reason=local-file-import`
+so a CMMC assessor can audit the decision from journald.
+
+`--skip-verify` bypasses the checksum step for catalog imports. The command
+still downloads via the catalog URL but does not compare the image digest
+against the sums file. Intended for narrow cases such as debugging upstream
+mirror issues or running against a transiently-broken `latest/` path; the
+skip is logged at WARN with `reason=skip-verify-flag` and printed to stderr
+so operators and assessors see it. Prefer `--file` with an out-of-band
+verified image over `--skip-verify` whenever possible.
+
+**Limitation:** verification confirms the image matches the digest the mirror
+served. A mirror compromise that swaps both image and sums file is not
+detected; closing that gap requires GPG signature verification of the sums
+file, deferred to a later phase.
+
+#### `spx admin images remove` caveats
+
+Admin-imported AMIs (`ImageOwnerAlias = "system"`) live
+under the `ami-<id>/` S3 prefix and use a viperblock-internal snap checkpoint
+at `snap-ami-<id>/` — there is no `snap-<id>/metadata.json`. The AWS handlers
+(`DeregisterImage`, `DeleteSnapshot`) reject system owners with
+`UnauthorizedOperation`, which is the right behaviour for tenant API callers
+but leaves no AWS-flow path to reclaim space. `spx admin images remove` is
+the admin-trust-boundary counterpart that performs the dependency walk and
+hard-deletes the blocks directly against predastore.
+
+`CopyImage` of a system AMI is metadata-only: it writes a fresh
+`snap-<acct>/metadata.json` whose `VolumeID` points at `ami-<sys>` and a new
+`ami-<acct>/config.json` referencing that snap. Volumes launched from the
+copied AMI read transitively from `ami-<sys>/chunks/...`. The remove command
+walks this transitive set and refuses if anything references the target.
+
+**TOCTOU window:** between the safety scan and the `config.json` delete a
+concurrent `RunInstances` against the AMI could create a new dependent
+volume. The window is sub-second on a healthy cluster. The admin running
+this command is expected to know the fleet's operational state; if the race
+fires the result is a `vol-<id>` with deleted backing blocks, recovered by
+terminating the orphaned instance.
+
+**`--force` bypasses every safety check** (dependents, ownership, missing /
+corrupt `config.json`). Use only for salvage of orphaned blocks. Running it
+against a live system AMI corrupts every dependent volume on the next disk
+read.
+
 ### GPU Management
 
 | Command | Flags | Description |
@@ -130,29 +194,20 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `monitor-instances` | — | `--instance-ids` | **NOT STARTED** |
 | `unmonitor-instances` | — | `--instance-ids` | **NOT STARTED** |
 
-`run-instances --iam-instance-profile` enforces `iam:PassRole` on the contained
-role ARN and rejects cross-account profile ARNs. Placement strategies: `spread`
-(1 per node, atomic CAS) and `cluster` (pin all to one node).
+### EC2 — Spot Instances
 
-`run-instances --capacity-reservation-specification` consumes a reserved slot on
-the node owning the targeted reservation (targeted-by-id only; `open`
-auto-matching is not supported). The combination with `--placement` (group) is
-rejected. `describe-instances` then reports the instance's `CapacityReservationId`.
+Spot Instance Requests (SIRs) are a **mock** over the on-demand `run-instances` path: a request synchronously launches real VMs on the operator's own compute and is then reported `active`/`fulfilled`. There is no spot market — no bidding, price rejection, interruption, or reclamation, and instances are never reclaimed.
 
-`modify-instance-metadata-options` is the only metadata-options mutator — AWS has
-no `get-instance-metadata-options` API, so the per-instance block is read through
-`describe-instances`. Only the hop limit is mutable; `--http-tokens optional` and
-the other posture-weakening values are refused with `UnsupportedOperation`, since
-the platform enforces IMDSv2 permanently. `run-instances --metadata-options` runs
-the same validation at launch.
+| Command | Implemented Flags | Missing Flags | Status |
+|---------|-------------------|---------------|--------|
+| `request-spot-instances` | `--instance-count` (default 1), `--type` (`one-time`/`persistent` — stored, behaviour identical), `--spot-price` (echoed only), `--client-token`, `--launch-specification` (ImageId, InstanceType, KeyName, SubnetId, SecurityGroupIds, UserData, BlockDeviceMappings, IamInstanceProfile, Placement.GroupName, NetworkInterfaces), `--tag-specifications` (spot-instances-request) | `--valid-from`, `--valid-until`, `--launch-group`, `--availability-zone-group`, `--block-duration-minutes`, `--instance-interruption-behavior`, `--dry-run` | **DONE** (mock) |
+| `describe-spot-instance-requests` | `--spot-instance-request-ids`, `--filters` (spot-instance-request-id, state, instance-id, launch.image-id, launch.instance-type, launch.key-name, type, launched-availability-zone, tag-key, tag:*) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
+| `cancel-spot-instance-requests` | `--spot-instance-request-ids` | `--dry-run` | **DONE** |
+
+`describe-spot-price-history` is **unsupported** (returns `InvalidAction`): on owned hardware there is no spot/on-demand price differential, so any synthetic price would be misleading rather than helpful.
 
 ### EC2 — IAM Instance Profile Associations
 
-Stored as `vm.VM.IamInstanceProfileArn` + `IamInstanceProfileAssociationId`
-(one ARN per instance). Association IDs (`iip-assoc-`) are regenerated on every
-Associate/Replace and never reused. Auto-disassociated on terminate; preserved
-across stop/start.
-
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
 | `associate-iam-instance-profile` | `--instance-id`, `--iam-instance-profile` (Name/Arn) | `--dry-run` | **DONE** |
@@ -164,10 +219,10 @@ across stop/start.
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-key-pair` | `--key-name`, `--key-type` (rsa/ed25519) | `--key-format`, `--tag-specifications`, `--dry-run` | **DONE** |
+| `create-key-pair` | `--key-name`, `--key-type` (rsa/ed25519), `--tag-specifications` | `--key-format`, `--dry-run` | **DONE** |
 | `describe-key-pairs` | `--key-names`, `--key-pair-ids`, `--filters` (key-pair-id, key-name, fingerprint, tag:*) | `--max-results`, `--dry-run` | **DONE** |
 | `delete-key-pair` | `--key-name`, `--key-pair-id` | `--dry-run` | **DONE** |
-| `import-key-pair` | `--key-name`, `--public-key-material` | `--tag-specifications`, `--dry-run` | **DONE** |
+| `import-key-pair` | `--key-name`, `--public-key-material`, `--tag-specifications` | `--dry-run` | **DONE** |
 
 ### EC2 — AMI Images
 
@@ -183,16 +238,14 @@ across stop/start.
 | `reset-image-attribute` | `--image-id`, `--attribute description` | `--attribute launchPermission`, `--dry-run` | **DONE** |
 | `import-image` | — | `--disk-containers`, `--description`, `--architecture`, `--platform` | **NOT STARTED** |
 
-`copy-image` is metadata-only — no block copy. The new snapshot inherits the
-source `VolumeID`. `register-image`/`copy-image` accept `uefi-preferred` as
-input but treat it as `uefi` at launch.
+`copy-image` is metadata-only — no block copy. The new snapshot inherits the. source `VolumeID`.
 
 ### EC2 — Volumes (EBS)
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
 | `describe-volumes` | `--volume-ids`, `--filters` (volume-id, status, size, volume-type, attachment.instance-id, attachment.status, attachment.device, availability-zone, tag:*), persisted `DeleteOnTermination` | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
-| `create-volume` | `--size`, `--availability-zone`, `--volume-type` (gp3), `--snapshot-id` | `--iops` (hardcoded 3000), `--encrypted` (hardcoded false), `--throughput`, `--tag-specifications` | **DONE** |
+| `create-volume` | `--size`, `--availability-zone`, `--volume-type` (gp3), `--snapshot-id`, `--tag-specifications` | `--iops` (hardcoded 3000), `--encrypted` (hardcoded false), `--throughput` | **DONE** |
 | `delete-volume` | `--volume-id` | `--dry-run` | **DONE** |
 | `modify-volume` | `--volume-id`, `--size`, `--volume-type`, `--iops` | `--throughput`, `--dry-run`, `--multi-attach-enabled` | **DONE** |
 | `attach-volume` | `--volume-id`, `--instance-id`, `--device` (auto-assigns `/dev/sd[f-p]`) | `--dry-run` | **DONE** |
@@ -228,8 +281,7 @@ input but treat it as `uefi` at launch.
 
 ### EC2 — Account Settings
 
-Persistence works (NATS JetStream KV) but stored values are not yet enforced
-by downstream services.
+Persistence works but stored values are not yet enforced by downstream services.
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
@@ -248,10 +300,7 @@ by downstream services.
 | `modify-instance-metadata-defaults` | — | `--http-tokens`, `--http-put-response-hop-limit`, `--http-endpoint`, `--instance-metadata-tags` | **NOT STARTED** |
 | `get-instance-metadata-defaults` | — | `--dry-run` | **NOT STARTED** |
 
-The account-level `modify-/get-instance-metadata-defaults` are blocked on an
-`aws-sdk-go` bump: the `InstanceMetadataDefaultsResponse` type is absent from the
-pinned v1.44.266 (these APIs landed ~March 2024), and the gateway's generic
-handler needs the typed input struct to route them.
+The account-level `modify-/get-instance-metadata-defaults` are blocked on an `aws-sdk-go` bump: the `InstanceMetadataDefaultsResponse` type is absent from our version (these APIs landed ~March 2024), and the gateway's generic handler needs the typed input struct to route them.
 
 ### EC2 — VPC Core
 
@@ -296,8 +345,7 @@ handler needs the typed input struct to route them.
 
 ### EC2 — Egress-Only Internet Gateway
 
-KV CRUD only — no OVN/OVS integration. EIGWs are stored but have no effect on
-network topology.
+KV CRUD only — no OVN/OVS integration. EIGWs are stored but have no effect on network topology.
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
@@ -309,7 +357,7 @@ network topology.
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-route-table` | `--vpc-id` | `--tag-specifications`, `--dry-run` | **DONE** |
+| `create-route-table` | `--vpc-id`, `--tag-specifications` | `--dry-run` | **DONE** |
 | `delete-route-table` | `--route-table-id` | `--dry-run` | **DONE** |
 | `describe-route-tables` | `--route-table-ids`, `--filters` (vpc-id, route-table-id, association.main, association.route-table-association-id, association.subnet-id, route.destination-cidr-block, route.gateway-id) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
 | `create-route` | `--route-table-id`, `--destination-cidr-block`, `--gateway-id`, `--nat-gateway-id` | `--egress-only-internet-gateway-id`, `--vpc-peering-connection-id`, `--dry-run` | **DONE** |
@@ -337,7 +385,7 @@ termination. Standalone attach/detach API is internal-only.
 
 ### EC2 — Elastic IP
 
-EIP commands are only registered when an external IPAM pool is configured.
+EIP handlers are always registered. Without a public IPAM pool (external mode disabled, or `nat` without a public pool), `describe-addresses` returns an empty list and mutating commands return `UnsupportedOperation`. In `nat` mode a public pool (`--external-pool` or `--external-source=dhcp` at init) enables the full EIP surface with host-delivered ingress.
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
@@ -352,7 +400,7 @@ EIP commands are only registered when an external IPAM pool is configured.
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-nat-gateway` | `--subnet-id`, `--allocation-id` | `--connectivity-type`, `--tag-specifications`, `--dry-run` | **DONE** |
+| `create-nat-gateway` | `--subnet-id`, `--allocation-id`, `--tag-specifications` | `--connectivity-type`, `--dry-run` | **DONE** |
 | `delete-nat-gateway` | `--nat-gateway-id` | `--dry-run` | **DONE** |
 | `describe-nat-gateways` | `--nat-gateway-ids`, `--filters` (vpc-id, state) | `--max-results`, `--next-token`, `--dry-run` | **DONE** |
 | `assign-private-nat-gateway-address` | — | `--nat-gateway-id`, `--private-ip-addresses` | **NOT STARTED** |
@@ -362,9 +410,9 @@ EIP commands are only registered when an external IPAM pool is configured.
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-placement-group` | `--group-name`, `--strategy` (spread/cluster) | `--partition-count`, `--spread-level`, `--tag-specifications`, `--dry-run` | **DONE** |
+| `create-placement-group` | `--group-name`, `--strategy` (spread/cluster), `--tag-specifications` | `--partition-count`, `--spread-level`, `--dry-run` | **DONE** |
 | `delete-placement-group` | `--group-name` | `--dry-run` | **DONE** |
-| `describe-placement-groups` | `--group-names`, `--group-ids`, `--filters` (strategy, state, spread-level, group-name) | `--dry-run` | **DONE** |
+| `describe-placement-groups` | `--group-names`, `--group-ids`, `--filters` (strategy, state, spread-level, group-name, tag:*, tag-key, tag-value) | `--dry-run` | **DONE** |
 
 ### EC2 — VPC Peering
 
@@ -480,8 +528,7 @@ EIP commands are only registered when an external IPAM pool is configured.
 
 ## IAM
 
-All IAM operations are account-scoped. Root user (account `000000000000`)
-bypasses policy evaluation entirely.
+All IAM operations are account-scoped. Root user (account `000000000000`) bypasses policy evaluation entirely.
 
 ### IAM — Users
 
@@ -491,6 +538,21 @@ bypasses policy evaluation entirely.
 | `get-user` | `--user-name` | — | **DONE** |
 | `list-users` | `--path-prefix` | `--max-items`, `--marker` | **DONE** |
 | `delete-user` | `--user-name` | — | **DONE** |
+| `update-user` | — | `--user-name`, `--new-path`, `--new-user-name` | **NOT STARTED** |
+| `put-user-policy` | `--user-name`, `--policy-name`, `--policy-document` | — | **DONE** |
+| `get-user-policy` | `--user-name`, `--policy-name` | — | **DONE** |
+| `delete-user-policy` | `--user-name`, `--policy-name` | — | **DONE** |
+| `list-user-policies` | `--user-name` | `--max-items`, `--marker` | **DONE** |
+| `tag-user` | `--user-name`, `--tags` | — | **DONE** |
+| `untag-user` | `--user-name`, `--tag-keys` | — | **DONE** |
+| `list-user-tags` | `--user-name` | `--max-items`, `--marker` | **DONE** |
+| `put-user-permissions-boundary` | — | `--user-name`, `--permissions-boundary` | **NOT STARTED** |
+| `delete-user-permissions-boundary` | — | `--user-name` | **NOT STARTED** |
+| `create-login-profile` | — | `--user-name`, `--password` | **NOT STARTED** |
+| `get-login-profile` | — | `--user-name` | **NOT STARTED** |
+| `update-login-profile` | — | `--user-name`, `--password` | **NOT STARTED** |
+| `delete-login-profile` | — | `--user-name` | **NOT STARTED** |
+| `change-password` | — | `--old-password`, `--new-password` | **NOT STARTED** |
 
 ### IAM — Access Keys
 
@@ -500,6 +562,7 @@ bypasses policy evaluation entirely.
 | `list-access-keys` | `--user-name` | `--max-items`, `--marker` | **DONE** |
 | `delete-access-key` | `--access-key-id`, `--user-name` | — | **DONE** |
 | `update-access-key` | `--access-key-id`, `--user-name`, `--status` (Active/Inactive) | — | **DONE** |
+| `get-access-key-last-used` | — | `--access-key-id` | **NOT STARTED** |
 
 ### IAM — Policies
 
@@ -514,6 +577,17 @@ bypasses policy evaluation entirely.
 | `attach-user-policy` | `--user-name`, `--policy-arn` | — | **DONE** |
 | `detach-user-policy` | `--user-name`, `--policy-arn` | — | **DONE** |
 | `list-attached-user-policies` | `--user-name` | `--path-prefix`, `--max-items`, `--marker` | **DONE** |
+| `create-policy-version` | — | `--policy-arn`, `--policy-document`, `--set-as-default` | **NOT STARTED** |
+| `delete-policy-version` | — | `--policy-arn`, `--version-id` | **NOT STARTED** |
+| `set-default-policy-version` | — | `--policy-arn`, `--version-id` | **NOT STARTED** |
+| `list-entities-for-policy` | — | `--policy-arn`, `--entity-filter`, `--path-prefix`, `--policy-usage-filter` | **NOT STARTED** |
+| `tag-policy` | `--policy-arn`, `--tags` | — | **DONE** |
+| `untag-policy` | `--policy-arn`, `--tag-keys` | — | **DONE** |
+| `list-policy-tags` | `--policy-arn` | `--max-items`, `--marker` | **DONE** |
+| `generate-service-last-accessed-details` | — | `--arn`, `--granularity` | **NOT STARTED** |
+| `get-service-last-accessed-details` | — | `--job-id` | **NOT STARTED** |
+| `get-service-last-accessed-details-with-entities` | — | `--job-id`, `--service-namespace` | **NOT STARTED** |
+| `list-policies-granting-service-access` | — | `--arn`, `--service-namespaces` | **NOT STARTED** |
 
 ### IAM — Roles
 
@@ -532,6 +606,12 @@ bypasses policy evaluation entirely.
 | `put-role-policy` | `--role-name`, `--policy-name`, `--policy-document` | — | **DONE** |
 | `get-role-policy` | `--role-name`, `--policy-name` | — | **DONE** (document returned as raw JSON, not URL-encoded) |
 | `delete-role-policy` | `--role-name`, `--policy-name` | — | **DONE** |
+| `put-role-permissions-boundary` | — | `--role-name`, `--permissions-boundary` | **NOT STARTED** |
+| `delete-role-permissions-boundary` | — | `--role-name` | **NOT STARTED** |
+| `tag-role` | `--role-name`, `--tags` | — | **DONE** |
+| `untag-role` | `--role-name`, `--tag-keys` | — | **DONE** |
+| `list-role-tags` | `--role-name` | `--max-items`, `--marker` | **DONE** |
+| `update-role-description` | — | `--role-name`, `--description` | **NOT STARTED** |
 
 ### IAM — Instance Profiles
 
@@ -544,6 +624,9 @@ bypasses policy evaluation entirely.
 | `delete-instance-profile` | `--instance-profile-name` | — | **DONE** |
 | `add-role-to-instance-profile` | `--instance-profile-name`, `--role-name` | — | **DONE** |
 | `remove-role-from-instance-profile` | `--instance-profile-name`, `--role-name` | — | **DONE** |
+| `tag-instance-profile` | `--instance-profile-name`, `--tags` | — | **DONE** |
+| `untag-instance-profile` | `--instance-profile-name`, `--tag-keys` | — | **DONE** |
+| `list-instance-profile-tags` | `--instance-profile-name` | `--max-items`, `--marker` | **DONE** |
 
 ### IAM — OIDC Providers
 
@@ -556,22 +639,28 @@ bypasses policy evaluation entirely.
 | `add-client-id-to-open-id-connect-provider` | — | `--open-id-connect-provider-arn`, `--client-id` | **NOT STARTED** |
 | `remove-client-id-from-open-id-connect-provider` | — | `--open-id-connect-provider-arn`, `--client-id` | **NOT STARTED** |
 | `update-open-id-connect-provider-thumbprint` | — | `--open-id-connect-provider-arn`, `--thumbprint-list` | **NOT STARTED** |
-| `tag-open-id-connect-provider` / `untag-open-id-connect-provider` | — | `--open-id-connect-provider-arn`, `--tags`/`--tag-keys` | **NOT STARTED** |
+| `tag-open-id-connect-provider` / `untag-open-id-connect-provider` | `--open-id-connect-provider-arn`, `--tags`/`--tag-keys` | — | **DONE** |
+| `list-open-id-connect-provider-tags` | `--open-id-connect-provider-arn` | `--max-items`, `--marker` | **DONE** |
 
 ### IAM — Groups
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-group` | — | `--group-name`, `--path` | **NOT STARTED** |
-| `get-group` | — | `--group-name` | **NOT STARTED** |
-| `list-groups` | — | `--path-prefix`, `--max-items`, `--marker` | **NOT STARTED** |
-| `delete-group` | — | `--group-name` | **NOT STARTED** |
-| `add-user-to-group` | — | `--group-name`, `--user-name` | **NOT STARTED** |
-| `remove-user-from-group` | — | `--group-name`, `--user-name` | **NOT STARTED** |
-| `list-groups-for-user` | — | `--user-name`, `--max-items`, `--marker` | **NOT STARTED** |
-| `attach-group-policy` | — | `--group-name`, `--policy-arn` | **NOT STARTED** |
-| `detach-group-policy` | — | `--group-name`, `--policy-arn` | **NOT STARTED** |
-| `list-attached-group-policies` | — | `--group-name`, `--path-prefix`, `--max-items`, `--marker` | **NOT STARTED** |
+| `create-group` | `--group-name`, `--path` | — | **DONE** |
+| `get-group` | `--group-name` | — | **DONE** |
+| `list-groups` | `--path-prefix` | `--max-items`, `--marker` | **DONE** |
+| `delete-group` | `--group-name` | — | **DONE** |
+| `update-group` | — | `--group-name`, `--new-path`, `--new-group-name` | **NOT STARTED** |
+| `add-user-to-group` | `--group-name`, `--user-name` | — | **DONE** |
+| `remove-user-from-group` | `--group-name`, `--user-name` | — | **DONE** |
+| `list-groups-for-user` | `--user-name` | `--max-items`, `--marker` | **DONE** |
+| `attach-group-policy` | `--group-name`, `--policy-arn` | — | **DONE** |
+| `detach-group-policy` | `--group-name`, `--policy-arn` | — | **DONE** |
+| `list-attached-group-policies` | `--group-name`, `--path-prefix` | `--max-items`, `--marker` | **DONE** |
+| `put-group-policy` | `--group-name`, `--policy-name`, `--policy-document` | — | **DONE** |
+| `get-group-policy` | `--group-name`, `--policy-name` | — | **DONE** |
+| `delete-group-policy` | `--group-name`, `--policy-name` | — | **DONE** |
+| `list-group-policies` | `--group-name` | `--max-items`, `--marker` | **DONE** |
 
 ---
 
@@ -588,7 +677,7 @@ bypasses policy evaluation entirely.
 | `get-federation-token` | — | `--name`, `--policy`, `--policy-arns`, `--duration-seconds`, `--tags` | **NOT STARTED** |
 | `decode-authorization-message` | — | `--encoded-message` | **NOT STARTED** |
 
-Trust policies stored on roles (`AssumeRolePolicyDocument`) reject `Condition`, `NotPrincipal`, `NotAction`, empty-string `Action` elements, and empty `Principal` blocks at write time (`MalformedPolicyDocument`) — v1 has no Condition evaluator so accepting them would silently allow.
+Trust policies (`AssumeRolePolicyDocument`) reject `NotPrincipal`, `NotAction`, empty-string `Action` elements, and empty `Principal` blocks at write time (`MalformedPolicyDocument`). `Condition` blocks are rejected except on `sts:AssumeRoleWithWebIdentity` with `StringEquals` (IRSA), which v1 evaluates at assume time (`{iss}:sub`, `{iss}:aud`); anything wider is rejected to avoid silent over-grant.
 
 ---
 
@@ -625,7 +714,7 @@ curl -i http://169.254.169.254/latest/meta-data/instance-id
 | `/latest/meta-data/ami-id` | GET | launch `ImageId` | **DONE** |
 | `/latest/meta-data/ami-launch-index` | GET | Per-instance launch index (`0..n-1`), contiguous on partial failure | **DONE** |
 | `/latest/meta-data/reservation-id` | GET | `DescribeInstances` `Reservation.ReservationId` | **DONE** |
-| `/latest/meta-data/instance-life-cycle` | GET | `on-demand` (Spot not modelled) | **DONE** |
+| `/latest/meta-data/instance-life-cycle` | GET | `spot` for a spot-launched instance, else `on-demand`; defaults to `on-demand` on a resolution miss (never 404, the leaf is always advertised) | **DONE** |
 | `/latest/meta-data/local-ipv4` | GET | `ENIRecord.PrivateIpAddress` (== request source IP) | **DONE** |
 | `/latest/meta-data/public-ipv4` | GET | EIP, else instance public IP; empty body if none | **DONE** |
 | `/latest/meta-data/public-hostname` | GET | Mirrors `public-ipv4`; 404 when no public IP | **DONE** |
@@ -651,7 +740,7 @@ curl -i http://169.254.169.254/latest/meta-data/instance-id
 | `/latest/meta-data/block-device-mapping/...` | GET | `ami`/`root`/`ebsN`/`ephemeralN` device map | **NOT STARTED** (404) |
 | `/latest/meta-data/placement/{group-name,partition-number,availability-zone-id,host-id}` | GET | Placement extras beyond `availability-zone`/`region` | **NOT STARTED** (404) |
 | `/latest/meta-data/instance-action` | GET | `none` unless interruptible instances ship | **NOT STARTED** (404) |
-| `/latest/meta-data/spot/{instance-action,termination-time}` | GET | Only meaningful once Spot is modelled | **NOT STARTED** (404) |
+| `/latest/meta-data/spot/{instance-action,termination-time}` | GET | 404 is the faithful steady state for a never-interrupted spot instance ("no action scheduled"); a 200 body would trigger interruption handling in pollers (AWS Node Termination Handler / Karpenter). Not advertised in the `spot/` listing | **DONE** (404 by contract) |
 
 ---
 

docs/DESIGN.md

@@ -49,7 +49,7 @@ Request routing (`spinifex/gateway/gateway.go`):
 
 1. **Authentication**: SigV4 middleware validates AWS credentials and resolves the account ID
 2. **Throttling**: Per-account+action token bucket rejects bursts post-auth
-3. **Service Detection**: Extracts service name (ec2, iam, account, elasticloadbalancing, spinifex) from the Authorization header
+3. **Service Detection**: Extracts service name from the Authorization header
 4. **Action Dispatch**: Routes to service-specific handler
 
 ```go
@@ -60,8 +60,20 @@ case "account":
     err = gw.Account_Request(w, r)
 case "iam":
     err = gw.IAM_Request(w, r)
+case "sts":
+    err = gw.STS_Request(w, r)
 case "elasticloadbalancing":
     err = gw.ELBv2_Request(w, r)
+case "eks":
+    err = gw.EKS_Request(w, r)
+case "ecs":
+    err = gw.ECS_Request(w, r)
+case "ecr":
+    err = gw.ECR_Request(w, r)
+case "acm":
+    err = gw.ACM_Request(w, r)
+case "tagging":
+    err = gw.Tagging_Request(w, r)
 case "spinifex":
     err = gw.Spinifex_Request(w, r)
 }
@@ -79,18 +91,12 @@ The EC2 handler (`spinifex/gateway/ec2.go`) parses the `Action` parameter and de
     return gateway_ec2_instance.DescribeInstances(input, gw.NATSConn, gw.DiscoverActiveNodes(), accountID)
 }),
 // ... + volumes, snapshots, VPCs, subnets, route tables, IGWs, NAT gateways,
-// security groups, network interfaces, placement groups, key pairs, images, tags
+// security groups, network interfaces, elastic IPs, placement groups, key pairs,
+// images, tags, capacity reservations, spot instances
 ```
 
 ### 4. NATS Messaging
 
-The full service-to-subject map — which service subscribes to what, what
-each one publishes downstream, what each depends on — is captured in
-[`service-interfaces.yaml`](./service-interfaces.yaml). It is the source
-of truth for targeted e2e suite selection (see
-`docs/development/improvements/e2e-targeted-suite-selection.md`) and is
-validated by `make manifest-check`.
-
 The gateway communicates with daemons via NATS request/response. Most calls go through `utils.NATSRequest`, which marshals the input, attaches the account ID as a NATS header, and unmarshals the typed response:
 
 ```go
@@ -114,7 +120,7 @@ Daemons (`spinifex/daemon/daemon.go`) subscribe to NATS topics and handle reques
 {"ec2.DescribeInstanceTypes", d.handleEC2DescribeInstanceTypes, ""},
 ```
 
-`ec2.RunInstances` is dynamic: `ResourceManager` subscribes to `ec2.RunInstances.{type}` (and `ec2.RunInstances.{type}.{nodeId}` for targeted dispatch) only while the node has capacity for that type, and unsubscribes when full. Per-instance commands flow over `ec2.cmd.{instanceID}`, subscribed only by the owning node.
+`ec2.RunInstances` is dynamic: `ResourceManager` subscribes to `ec2.RunInstances.{type}` only while the node has capacity for that type, and unsubscribes when full. The node-targeted subject `ec2.RunInstances.{type}.{nodeId}` stays subscribed even at full capacity — the launch-time `allocate()` enforces limits there. Per-instance commands flow over `ec2.cmd.{instanceID}`, subscribed only by the owning node.
 
 **Queue Groups**: topics with the `spinifex-workers` queue group are load-balanced — only one daemon handles each request. Topics without a queue group fan out to all daemons.
 
@@ -151,7 +157,7 @@ type Daemon struct {
     config        *config.Config          // Node-specific configuration
     natsConn      *nats.Conn              // NATS connection
     resourceMgr   *ResourceManager        // CPU/Memory tracking + dynamic RunInstances subscriptions
-    Instances     vm.Instances            // Local VMs
+    vmMgr         *vm.Manager             // Local VMs
     // ... + one service struct per EC2/ELBv2 resource
     //       (instance, key, image, volume, snapshot, tags, vpc, subnet,
     //        igw, eigw, natgw, routetable, eip, placementgroup, elbv2, ...)
@@ -172,6 +178,8 @@ type ResourceManager struct {
     allocatedVCPU int
     allocatedMem  float64
     instanceTypes map[string]*ec2.InstanceTypeInfo   // t3.micro, t3.small, etc.
+    // ... + capacity-reservation accounting, GPU manager, nbdkit memory
+    //       overhead tracking, and the dynamic-subscription state
 }
 ```
 
@@ -236,22 +244,26 @@ Cluster configuration (`spinifex/config/config.go`):
 
 ```go
 type ClusterConfig struct {
-    Epoch   uint64            // Incremented on config changes
-    Node    string            // This node's identifier
-    Version string            // Spinifex version
-    Nodes   map[string]Config // Configuration for all nodes
+    Epoch     uint64            // Incremented on config changes
+    Node      string            // This node's identifier
+    Version   string            // Spinifex version
+    Network   NetworkConfig     // Cluster-wide external network settings (IP pools, IPsec)
+    Bootstrap BootstrapConfig   // Default VPC IDs written by admin init
+    AWS       AWSConfig         // Cluster-wide AWS-parity settings (region, endpoint suffix)
+    Nodes     map[string]Config // Configuration for all nodes
 }
 
 type Config struct {
-    Node, Host, Region, AZ string
-    BaseDir, DataDir       string
+    Node, Host, AdvertiseIP  string
+    Region, AZ               string
+    BaseDir, DataDir, WalDir string
+    Services   []string         // Which services this node runs locally
     Daemon     DaemonConfig
     NATS       NATSConfig       // Broker address, JetStream, ACL token, CA cert
     AWSGW      AWSGWConfig      // Gateway host, TLS certs, throttle config
     Predastore PredastoreConfig // S3 endpoint + service credentials
     Viperblock ViperblockConfig
     VPCD       VPCDConfig       // OVN/OVS endpoints
-    Network    NetworkConfig
 }
 ```
 
@@ -263,14 +275,14 @@ Configs are generated by `spx admin init` (leader) or `spx admin join` (follower
 spinifex/
 ├── spinifex/
 │   ├── services/                # Long-running services (awsgw, spinifex, viperblockd,
-│   │                            #   predastore, nats, vpcd, spinifexui)
+│   │                            #   predastore, nats, spinifexui)
 │   ├── gateway/                 # AWS API surface
 │   │   ├── gateway.go           # chi router, SigV4 auth, throttling
 │   │   ├── auth.go              # SigV4 verification + IAM lookup
 │   │   ├── ec2.go               # EC2 action dispatcher
 │   │   ├── ec2/                 # Per-resource handlers (instance, volume, vpc, ...)
 │   │   ├── elbv2/, iam/, policy/
-│   ├── handlers/                # NATS-side service implementations (ec2, elbv2, iam)
+│   ├── handlers/                # NATS-side service implementations (ec2, elbv2, iam, ...)
 │   ├── daemon/
 │   │   ├── daemon.go            # Daemon entry, NATS subscription table
 │   │   └── daemon_handlers_*.go # One file per resource family
@@ -279,11 +291,13 @@ spinifex/
 │   ├── instancetypes/           # t3.*, m5.*, sys.* definitions
 │   ├── lbagent/                 # In-VM agent for ELB target health
 │   ├── vm/                      # VM instance representation
+│   ├── vpcd/                    # VPC daemon (OVN/OVS network programming)
 │   └── qmp/                     # QEMU Machine Protocol client
 └── cmd/
     ├── spinifex/                # spx CLI
     ├── lb-agent/                # ELB target agent binary
-    └── installer/               # Platform installer
+    ├── installer/               # Platform installer
+    └── ...                      # ECS agent, ECR credential provider, EKS helper binaries
 ```
 
 ## Development

docs/TAG-CONVENTIONS.md

@@ -1,11 +1,7 @@
 # Spinifex Tag Conventions
 
-Spinifex uses a small set of reserved tag keys to mark resources that are
-owned and managed by the platform itself (as opposed to resources created
-directly by customer AWS API calls). The UI uses these tags to hide
-system-owned resources from customer-facing listings so operators do not
-mistake them for user workloads and accidentally terminate or modify
-them.
+Spinifex uses a small set of reserved tag keys to mark resources that are owned and managed by the platform itself (as opposed to resources created directly by customer AWS API calls). The UI uses these tags to hide system-owned resources from customer-facing listings so operators do not
+mistake them for user workloads and accidentally terminate or modify them.
 
 ## `spinifex:managed-by`
 
@@ -17,50 +13,20 @@ Identifies the Spinifex platform component that owns a resource.
 
 The tag is applied by the backend at create/launch time:
 
-- **Instance** — set in
-  `spinifex/spinifex/daemon/daemon_system_instance.go` when
-  `LaunchSystemInstance` builds the `RunInstancesInput`, and mirrored onto
-  the stored `vm.VM.ManagedBy` field for fast lookup.
-- **ENI** — set in
-  `spinifex/spinifex/handlers/elbv2/service_impl.go` when
-  `CreateLoadBalancer` calls `CreateNetworkInterface`.
-- **AMI** — declared on the catalog entry in
-  `spinifex/spinifex/utils/images.go` and copied into
-  `AMIMetadata.Tags` by the image importer in
-  `spinifex/cmd/spinifex/cmd/admin.go`.
+- **Instance** — set in `spinifex/spinifex/daemon/daemon_system_instance.go` when `LaunchSystemInstance` builds the `RunInstancesInput`, and mirrored onto the stored `vm.VM.ManagedBy` field for fast lookup.
+- **ENI** — set in `spinifex/spinifex/handlers/elbv2/service_impl.go` when `CreateLoadBalancer` calls `CreateNetworkInterface`.
+- **AMI** — declared on the catalog entry in `spinifex/spinifex/utils/images.go` and copied into `AMIMetadata.Tags` by the image importer in `spinifex/cmd/spinifex/cmd/admin.go`.
 
-Tag keys and values are defined once in `spinifex/spinifex/tags/tags.go`;
-new system components should register their value there and reuse the
-constants rather than hardcoding strings.
+Tag keys and values are defined once in `spinifex/spinifex/tags/tags.go`; new system components should register their value there and reuse the constants rather than hardcoding strings.
 
 ## UI behaviour
 
-The admin **Nodes** page and the **Images** (AMIs) page filter out
-resources that carry `spinifex:managed-by`. Engineers who need to see
-system resources for debugging can append `?system=1` to the URL — the
-filter is bypassed and a hidden-count banner is replaced with the full
-listing. Direct navigation to a system AMI detail page without
-`?system=1` renders the "Image not found" fallback.
+The admin **Nodes** page and the **Images** (AMIs) page filter out resources that carry `spinifex:managed-by`. Engineers who need to see system resources for debugging can append `?system=1` to the URL — the filter is bypassed and a hidden-count banner is replaced with the full listing. Direct navigation to a system AMI detail page without `?system=1` renders the "Image not found" fallback.
 
-The EC2 **Instances** page does not need UI-side filtering: backend
-SigV4 authentication scopes `DescribeInstances` by account ID, and
-system instances live under the global system account
+The EC2 **Instances** page does not need UI-side filtering: backend SigV4 authentication scopes `DescribeInstances` by account ID, and system instances live under the global system account
 (`000000000000`), so customers never see them.
 
 ## `spinifex:lb-arn`
 
-Applied to ELBv2-managed ENIs alongside `spinifex:managed-by=elbv2`.
-Stores the ARN of the parent load balancer so reconciliation can map an
-ENI back to its owner without a separate lookup. Customer-facing
-filtering keys off `spinifex:managed-by`; `spinifex:lb-arn` is
-informational.
-
-## Adding a new system component
-
-1. Add a new value constant next to `ManagedByELBv2` in
-   `spinifex/spinifex/tags/tags.go`.
-2. Tag every resource the component creates (instance, ENI, AMI,
-   volume, …) with `spinifex:managed-by=<value>`.
-3. Add the value to the table above so operators can recognise it.
-4. No frontend changes are required — the UI filters on the key, not
-   the value.
+Applied to ELBv2-managed ENIs alongside `spinifex:managed-by=elbv2`. Stores the ARN of the parent load balancer so reconciliation can map an ENI back to its owner without a separate lookup. Customer-facing
+filtering keys off `spinifex:managed-by`; `spinifex:lb-arn` is informational.

docs/admin/setting-up-your-cluster/README.md

@@ -1,7 +1,7 @@
 ---
 title: "Setting Up Your Cluster"
 description: "Import an AMI, create a VPC with a public subnet, and launch your first EC2 instance on Spinifex."
-category: "Getting Started"
+category: "Admin"
 tags:
   - setup
   - vpc
@@ -33,16 +33,14 @@ resources:
 
 ## Overview
 
-This guide assumes Spinifex is already installed and running. If not, follow one of the installation guides first:
-- [Single-Node Install](/docs/install)
-- [Multi-Node Install](/docs/install-multi-node)
-- [Source Install](/docs/install-source)
+This guide walks through the first steps on a freshly installed cluster: importing an AMI, creating an SSH key and a VPC with a public subnet, launching an instance, and connecting to it.
 
 ## Instructions
 
 ## Prerequisites
 
-Ensure the AWS profile is set:
+- Spinifex installed and running — see [Single-Node Install](/docs/install), [Multi-Node Install](/docs/install-multi-node), or [Source Install](/docs/install-source)
+- AWS CLI configured with the `spinifex` profile:
 
 ```bash
 export AWS_PROFILE=spinifex

docs/admin/spinifex-admin-cli/README.md

@@ -1,7 +1,7 @@
 ---
 title: "Spinifex Admin CLI"
 description: "Complete reference for the Spinifex administration CLI. Manage accounts, nodes, VMs, and services."
-category: "Administration"
+category: "Admin"
 tags:
   - cli
   - admin
@@ -25,30 +25,56 @@ resources:
 
 ## Overview
 
-The `spx` binary is the central administration tool for managing your Spinifex infrastructure. It provides commands for cluster initialization, account management, node operations, VM lifecycle, and service control.
+The `spx` binary is the central administration tool for managing your Spinifex infrastructure. It provides commands for cluster initialization, account management, node operations, VM lifecycle, and service control. All services in the Spinifex platform are managed through this single binary.
 
-**Binary location:** `/usr/local/bin/spx` (binary install) or `~/Development/mulga/spinifex/bin/spx` (source install)
-
-All services in the Spinifex platform are managed through this single binary.
+**Binary location:** `/usr/local/bin/spx`
 
 ## Instructions
 
 ## Account Management
 
-Create a new isolated account with an admin user and AdministratorAccess policy:
+Create a new isolated account. This provisions a sequential 12-digit account ID, an `admin` user with an `AdministratorAccess` policy attached, and an access key pair. The credentials are written to `~/.aws/credentials` and `~/.aws/config` under a `spinifex-<name>` profile automatically.
 
 ```bash
 spx admin account create --name myteam
+```
+
+```
+Account created successfully!
+  Account ID:        000000000002
+  Account Name:      myteam
+  Admin User:        admin
+  Access Key ID:     AKIA1A2B3C4D5E6F7890ABCD
+  Secret Access Key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+  AWS Profile:       spinifex-myteam
+
+Use with:
+  AWS_PROFILE=spinifex-myteam aws ec2 describe-instances
+```
+
+> **The secret access key is only shown once.** It is saved to `~/.aws/credentials` on the node where you ran the command; copy it from there if you need it elsewhere.
+
+Set the profile to start using the account:
+
+```bash
 export AWS_PROFILE=spinifex-myteam
 ```
 
+To create additional users and scoped permissions within the account, see [IAM Users and Policies](/docs/iam-users-and-policies).
+
 List all accounts:
 
 ```bash
 spx admin account list
 ```
 
-
+```
+ACCOUNT ID     NAME                 STATUS     CREATED
+----------     ----                 ------     -------
+000000000000   system               ACTIVE     2026-07-03 02:17
+000000000001   spinifex             ACTIVE     2026-07-03 02:17
+000000000002   myteam               ACTIVE     2026-07-03 03:32
+```
 
 ## Node Management
 
@@ -97,7 +123,7 @@ configuration is needed.
 
 ## Cluster Shutdown
 
-Coordinated, phased shutdown of the entire cluster (API/UI → VMs → viperblock → predastore → NATS/daemon):
+Coordinated, phased shutdown of the entire cluster — running VMs are stopped before storage and control-plane services:
 
 ```bash
 spx admin cluster shutdown

docs/admin/update/README.md

@@ -1,7 +1,7 @@
 ---
 title: "Updating Spinifex"
 description: "Upgrade an existing Spinifex installation to the latest release."
-category: "Administration"
+category: "Admin"
 tags:
   - update
   - upgrade

docs/administration/creating-accounts/README.md

@@ -1,55 +0,0 @@
----
-title: "Creating Accounts"
-description: "Create and manage isolated user accounts with their own resources."
-category: "Administration"
-tags:
-  - accounts
-  - iam
-  - admin
-resources:
-  - title: "Spinifex Repository"
-    url: "https://github.com/mulgadc/spinifex"
----
-
-# Creating Accounts
-
-> Create and manage isolated user accounts with their own resources.
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Instructions](#instructions)
-- [Troubleshooting](#troubleshooting)
-
----
-
-## Overview
-
-Spinifex supports multi-tenant account isolation. Each account gets its own IAM credentials, AWS CLI profile, and isolated resource namespace.
-
-## Instructions
-
-## Create Account
-
-```bash
-spx admin account create --name myteam
-export AWS_PROFILE=spinifex-myteam
-```
-
-## Troubleshooting
-
-## Credentials Not Working
-
-Verify the AWS CLI configuration files exist and contain the correct profile:
-
-```bash
-cat ~/.aws/config
-cat ~/.aws/credentials
-```
-
-Ensure the `AWS_PROFILE` environment variable matches the profile name:
-
-```bash
-echo $AWS_PROFILE
-export AWS_PROFILE=spinifex-myteam
-```

docs/compute/gpu-passthrough/README.md

@@ -72,7 +72,7 @@ After setup and reboot, enable GPU passthrough:
 sudo spx admin gpu enable
 ```
 
-The command writes the configuration, signals the daemon (`SIGHUP`), and waits up to 30 seconds for the daemon to confirm the new state. On success it prints the current GPU status.
+The command writes the configuration, notifies the daemon and waits up to 30 seconds for the daemon to confirm the new state. On success it prints the current GPU status.
 
 <img src="../../../.github/assets/images/gpu-enabled.png" alt="GPU enabled">
 

docs/compute/launching-instances/README.md

@@ -22,11 +22,12 @@ resources:
 - [Overview](#overview)
 - [Launch](#launch)
 - [Manage](#manage)
-- [Reboot](#reboot)
 - [Modify Instance Attributes](#modify-instance-attributes)
+- [Instance Metadata Options](#instance-metadata-options)
+- [Spot Instances](#spot-instances)
 - [Console Output](#console-output)
 - [Instance Types](#instance-types)
-- [SSH (Development)](#ssh-development)
+- [SSH](#ssh)
 - [Troubleshooting](#troubleshooting)
   - [Instance Fails to Boot](#instance-fails-to-boot)
   - [Cannot SSH Into Instance](#cannot-ssh-into-instance)
@@ -37,22 +38,12 @@ resources:
 
 Spinifex provides EC2-compatible VM management built on QEMU/KVM. Instances support cloud-init, SSH key injection, VPC networking, and standard AWS lifecycle operations.
 
-**Supported operations:**
-
-- `run-instances` — Launch new VMs
-- `describe-instances` — Query state
-- `stop-instances` / `start-instances` — Lifecycle
-- `reboot-instances` — In-place restart (QMP reset)
-- `terminate-instances` — Permanent removal
-- `modify-instance-attribute` — Change type, user data, or source/dest check
-- `get-console-output` — Retrieve serial console log
-- `describe-instance-types` — List available instance types
-
 ## Instructions
 
 ## Prerequisites
 
-Ensure the AWS profile is set:
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:
 
 ```bash
 export AWS_PROFILE=spinifex
@@ -61,18 +52,20 @@ export AWS_PROFILE=spinifex
 ## Launch
 
 ```bash
-aws ec2 run-instances \
+INSTANCE_ID=$(aws ec2 run-instances \
   --image-id $SPINIFEX_AMI \
   --instance-type t3.small \
-  --key-name spinifex-key
-
-export INSTANCE_ID="i-XXX"
+  --key-name spinifex-key \
+  --query 'Instances[0].InstanceId' --output text)
 ```
 
+Other supported launch flags: `--subnet-id`, `--security-group-ids`, `--tag-specifications`, `--user-data`, `--iam-instance-profile`, `--block-device-mappings`, `--placement`, `--count`.
+
 ## Manage
 
 ```bash
 aws ec2 describe-instances --instance-ids $INSTANCE_ID
+aws ec2 describe-instance-status --instance-ids $INSTANCE_ID --include-all-instances
 aws ec2 stop-instances --instance-ids $INSTANCE_ID
 aws ec2 start-instances --instance-ids $INSTANCE_ID
 aws ec2 terminate-instances --instance-ids $INSTANCE_ID
@@ -81,7 +74,7 @@ aws ec2 reboot-instances --instance-ids $INSTANCE_ID
 
 ## Modify Instance Attributes
 
-Change instance type, user data, or source/dest check. Instance type and user data require the instance to be **stopped** first.
+Change instance type, user data, or termination protection. Instance type and user data require the instance to be **stopped** first.
 
 ### Change Instance Type
 
@@ -95,6 +88,37 @@ aws ec2 modify-instance-attribute \
 aws ec2 start-instances --instance-ids $INSTANCE_ID
 ```
 
+### Termination Protection
+
+```bash
+aws ec2 modify-instance-attribute \
+  --instance-id $INSTANCE_ID \
+  --disable-api-termination
+```
+
+## Instance Metadata Options
+
+IMDSv2 is always enforced — `--http-tokens optional` and `--http-endpoint disabled` are rejected with `UnsupportedOperation`, matching AWS account-level IMDSv2 enforcement. The hop limit is adjustable (raise it for containerised workloads that reach IMDS through an extra network hop):
+
+```bash
+aws ec2 modify-instance-metadata-options \
+  --instance-id $INSTANCE_ID \
+  --http-put-response-hop-limit 2
+```
+
+## Spot Instances
+
+Spot requests are supported as a compatibility layer over the on-demand path: requests launch real VMs on your own compute immediately and report `active`/`fulfilled`. There is no spot market — no bidding, interruption, or reclamation.
+
+```bash
+aws ec2 request-spot-instances \
+  --instance-count 1 \
+  --launch-specification '{"ImageId":"'$SPINIFEX_AMI'","InstanceType":"t3.small","KeyName":"spinifex-key"}'
+
+aws ec2 describe-spot-instance-requests
+aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $SIR_ID
+```
+
 ## Console Output
 
 Retrieve the serial console log for a running instance. Output is base64-encoded.
@@ -121,7 +145,8 @@ aws ec2 describe-instance-types
 Filter to a specific type:
 
 ```bash
-aws ec2 describe-instance-types --instance-types t3.micro t3.small
+aws ec2 describe-instance-types \
+  --query "InstanceTypes[?InstanceType=='t3.small']"
 ```
 
 Show capacity (how many of each type can still be launched):
@@ -131,16 +156,9 @@ aws ec2 describe-instance-types \
   --filters Name=capacity,Values=true
 ```
 
-## SSH (Development)
+## SSH
 
-In development mode, find the QEMU port forward and connect via localhost:
-
-```bash
-ps auxw | grep $INSTANCE_ID
-ssh -i ~/.ssh/spinifex-key ubuntu@127.0.0.1 -p <port>
-```
-
-For production SSH via public IPs, see [Setting Up Your Cluster](/docs/setting-up-your-cluster).
+To SSH into instances via their public IPs, see [Setting Up Your Cluster](/docs/setting-up-your-cluster).
 
 ## Troubleshooting
 

docs/compute/placement-groups/README.md

@@ -50,7 +50,8 @@ The `partition` strategy is not supported.
 
 ## Prerequisites
 
-Ensure the AWS profile is set:
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:
 
 ```bash
 export AWS_PROFILE=spinifex

docs/compute/vpc-networking/README.md

@@ -191,11 +191,12 @@ is identical:
 Both support the same AWS features: public subnets, Elastic IPs, security groups,
 DescribeInstances showing public IPs.
 
-## `nat` — Outbound Only (Simple)
+## `nat` — Shared SNAT (Simple)
 
-All VMs share a single external IP for outbound SNAT. No public IPs, no Elastic
-IPs, no inbound from WAN. All subnets behave as private subnets with internet
-access.
+All VMs share a single external IP for outbound SNAT. By default there are no
+public IPs, no Elastic IPs, and no inbound from WAN — all subnets behave as
+private subnets with internet access. On routed-NAT nodes, adding a public pool
+restores full public IP parity (see below).
 
 The `gateway_ip` is the IP that OVN uses for SNAT. You can set it statically or
 use `setup-ovn.sh --dhcp` to obtain one from the router. This is the router's
@@ -215,6 +216,90 @@ gateway_ip = "192.168.1.100"     # Single IP for all VM outbound SNAT
 prefix_len = 24
 ```
 
+### Host access to instances (jumpbox pattern)
+
+The spinifex host automatically reaches every instance's **private IP**: IGW
+attach installs a host route into OVN (`<vpc-cidr> via <gateway-transit-ip>
+dev spx-nat-host`) and exempts the transit net from SNAT, so replies to
+host-initiated connections come back un-NATted. No per-instance setup.
+
+Security groups still apply and the default SG is closed to the host, same as
+AWS — open SSH/ICMP from the transit net first:
+
+```bash
+aws ec2 authorize-security-group-ingress --group-id $SG \
+  --protocol tcp --port 22 --cidr 100.127.0.0/24
+```
+
+Then use the host as a jumpbox for remote access:
+
+```bash
+ssh -J admin@<spinifex-host> ubuntu@<instance-private-ip>
+```
+
+Extra networks that must reach instances without SNAT (e.g. a management LAN)
+can be added via `[network] nat_exempt_cidrs = ["192.168.50.0/24"]`.
+
+### Public IPs in NAT mode (public pool)
+
+A routed-NAT node (`setup-ovn.sh --nat-uplink` + `spx admin init
+--external-mode=nat`) can carry a public pool alongside the internal
+`nat-transit` pool. With one configured, nat mode behaves like pool mode for
+public IPs: `MapPublicIpOnLaunch` on the default subnet, auto-assigned public
+IPs, and Elastic IPs all work. Spinifex delivers each public IP at the host —
+a `/32` route steers it into OVN and a proxy-ARP neighbor entry answers for it
+on the uplink (L3 only, same MAC, so it works on WiFi and other non-bridgeable
+uplinks).
+
+Static range carved out of the router's DHCP scope:
+
+```bash
+spx admin init --external-mode=nat \
+  --external-pool 192.168.1.150-192.168.1.250 \
+  --external-gateway 192.168.1.1
+```
+
+Or lease public IPs from the upstream router's DHCP:
+
+```bash
+spx admin init --external-mode=nat --external-source=dhcp \
+  --external-bind-bridge wlan0
+```
+
+On WiFi/WWAN uplinks the leases are requested with the interface's own MAC
+(`dhcp_mac = "interface"`, written automatically) and distinguished by DHCP
+client-id. Some routers key leases by MAC and ignore the client-id — Spinifex
+detects this (the router hands the same IP to two client-ids) and fails the
+allocation with advice to switch to a static range.
+
+Resulting config:
+
+```toml
+[network]
+external_mode = "nat"
+bridge_mode   = "nat"
+
+[[network.external_pools]]
+name       = "nat-transit"        # internal transit net (auto-generated)
+gateway    = "100.127.0.1"
+prefix_len = 24
+
+[[network.external_pools]]
+name        = "wan"               # public pool
+range_start = "192.168.1.150"
+range_end   = "192.168.1.250"
+gateway     = "192.168.1.1"
+prefix_len  = 24
+```
+
+**Caveat — reaching an EIP from the spinifex host itself.** Host-sourced
+traffic enters OVN from the transit net, which is exempt from NAT (that is what
+makes the jumpbox pattern work) — so a host connection to an EIP that carries
+the transit source IP would skip DNAT. Spinifex stamps the EIP route with the
+uplink's LAN IP as source to avoid this, but if no uplink address can be
+determined, connect to the instance's **private IP** from the host instead.
+Other machines on the LAN are unaffected.
+
 ## Disabled (Empty/Omitted)
 
 VPC networking is overlay-only. No external connectivity. Instances can only
@@ -222,19 +307,21 @@ communicate within their VPC.
 
 ## Mode Comparison
 
-| Capability                        | `pool` (static) | `pool` (dhcp) | `nat`    | Disabled |
-| --------------------------------- | --------------- | ------------- | -------- | -------- |
-| Outbound internet                 | Yes             | Yes           | Yes      | No       |
-| Inbound from WAN                  | Yes (1:1 NAT)   | Yes (1:1 NAT) | No       | No       |
-| Public subnets                    | Yes             | Yes           | No       | No       |
-| Auto-assign public IPs            | Yes             | Yes           | No       | No       |
-| Elastic IPs                       | Yes             | Yes           | No       | No       |
-| DescribeInstances shows public IP | Yes             | Yes           | No       | No       |
-| Admin must reserve IP range       | Yes             | No            | No       | No       |
-| Needs router DHCP                 | No              | Yes           | Optional | No       |
+| Capability                        | `pool` (static) | `pool` (dhcp) | `nat`             | Disabled |
+| --------------------------------- | --------------- | ------------- | ----------------- | -------- |
+| Outbound internet                 | Yes             | Yes           | Yes               | No       |
+| Host reaches instance private IPs | No              | No            | Yes (routed)      | No       |
+| Inbound from WAN                  | Yes (1:1 NAT)   | Yes (1:1 NAT) | With public pool  | No       |
+| Public subnets                    | Yes             | Yes           | With public pool  | No       |
+| Auto-assign public IPs            | Yes             | Yes           | With public pool  | No       |
+| Elastic IPs                       | Yes             | Yes           | With public pool  | No       |
+| DescribeInstances shows public IP | Yes             | Yes           | With public pool  | No       |
+| Admin must reserve IP range       | Yes             | No            | Only static pool  | No       |
+| Needs router DHCP                 | No              | Yes           | Optional          | No       |
 
-If you start with `nat` and later need public subnets, switch to `pool` and
-define a range (or use `source = "dhcp"`) — no data migration needed.
+If you start with `nat` and later need public subnets: on a bridgeable uplink
+switch to `pool` and define a range (or use `source = "dhcp"`); on a routed-NAT
+node just add a public pool alongside `nat-transit` — no data migration needed.
 
 ## Bridge Setup — Physical Network Wiring
 
@@ -452,7 +539,7 @@ dns_servers = ["8.8.8.8"]           # DNS for VMs (optional)
 
 | Field         | Required    | Description                                                                                                                                                            |
 | ------------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `name`        | Yes         | Unique pool name. Used as NATS KV key and in `AllocateAddress`.                                                                                                        |
+| `name`        | Yes         | Unique pool name. Referenced by `AllocateAddress` to target a specific pool.                                                                                           |
 | `source`      | No          | IP source: `"static"` (default) uses `range_start`/`range_end`. `"dhcp"` obtains IPs from the router's DHCP server on each VM launch.                                  |
 | `range_start` | Static only | First IP in the range. First IP is reserved for OVN gateway SNAT (unless `gateway_ip` overrides).                                                                      |
 | `range_end`   | Static only | Last IP in the range.                                                                                                                                                  |
@@ -483,13 +570,16 @@ These are different things:
 
 ```toml
 [nodes.spx1.vpcd]
-ovn_nb_addr        = "tcp:10.1.3.181:6641"   # OVN Northbound DB
-ovn_sb_addr        = "tcp:10.1.3.181:6642"   # OVN Southbound DB
+# Comma-separated list of the OVN NB/SB quorum endpoints (3 DB nodes). vpcd and
+# ovn-controller fail over across them and follow the RAFT leader.
+ovn_nb_addr        = "tcp:10.1.3.181:6641,tcp:10.1.3.182:6641,tcp:10.1.3.183:6641"
+ovn_sb_addr        = "tcp:10.1.3.181:6642,tcp:10.1.3.182:6642,tcp:10.1.3.183:6642"
 external_interface = "br-wan"                 # WAN Linux bridge name
 ```
 
 | Field                | Description                                                                                                                                                                          |
 | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `ovn_nb_addr` / `ovn_sb_addr` | OVN Northbound / Southbound DB endpoint(s). The NB and SB databases run clustered via OVSDB RAFT across the first three nodes (client ports 6641/6642, RAFT ports 6643/6644). Each node's config lists all three quorum endpoints so the loss of one DB node does not stall the control plane. A single `tcp:IP:6641` string remains valid for single-node dev. |
 | `external_interface` | Linux bridge that owns the WAN uplink on this node (e.g. `br-wan`, `br-public`). The physical NIC is enslaved to this bridge — not configured here. Different nodes may differ. |
 
 ## Pool Selection Logic
@@ -506,26 +596,10 @@ When an instance needs a public IP:
 
 ## IPAM Storage
 
-Each pool gets a NATS KV entry in bucket `spinifex-external-ipam`, keyed by pool
-name. Allocation uses CAS (Compare-And-Set) for lock-free concurrent access:
-
-```json
-{
-  "pool_name": "wan",
-  "range_start": "192.168.1.150",
-  "range_end": "192.168.1.250",
-  "allocated": {
-    "192.168.1.150": { "type": "gateway" },
-    "192.168.1.151": {
-      "type": "auto_assign",
-      "eni_id": "eni-abc",
-      "instance_id": "i-123"
-    }
-  }
-}
-```
-
-Pools are initialized from `spinifex.toml` on vpcd startup (idempotent).
+Pool allocation state is stored durably in the cluster (NATS KV bucket
+`spinifex-external-ipam`, one entry per pool) and survives restarts. Each
+allocation records the ENI and instance holding the address. Pools are
+initialized from `spinifex.toml` on vpcd startup (idempotent).
 
 ## Deployment Examples
 
@@ -1046,6 +1120,27 @@ sudo ovn-trace ext-vpc-{vpcId} \
    arp.op==1 && arp.spa==192.168.1.13 && arp.tpa==192.168.1.201'
 ```
 
+### OVN DB RAFT cluster (NB/SB replication)
+
+The NB and SB databases run clustered across the first three nodes via native
+OVSDB RAFT. A 3-node quorum tolerates the loss of one DB node; check status when
+the control plane stalls.
+
+```bash
+# NB cluster status: expect 3 servers and exactly one "Role: leader"
+sudo ovn-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound
+
+# SB cluster status
+sudo ovn-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound
+
+# Drive a CLI at the whole quorum (fails over past a dead node)
+sudo ovn-nbctl --db=tcp:10.1.3.181:6641,tcp:10.1.3.182:6641,tcp:10.1.3.183:6641 show
+```
+
+A node showing only itself under `Servers:` never joined the cluster — confirm
+it was bootstrapped with `--db-cluster-remote-addr` pointing at the creator and
+that RAFT ports 6643/6644 are reachable between DB nodes.
+
 ### OVS (datapath / physical wiring)
 
 ```bash
@@ -1314,12 +1409,10 @@ OVNSB commit failed, force recompute next time.
 Repeated millions of times. Port binding never happens (`up: false`).
 
 **Cause:** Stale entries in the OVN Southbound DB (old chassis records, port
-bindings, datapath bindings) conflict with ovn-controller's expected state.
-Happens when `reset-dev-env.sh` cleans NB but not SB, or after ungraceful
-shutdowns.
+bindings, datapath bindings) conflict with ovn-controller's expected state,
+typically after an ungraceful shutdown.
 
-**Fix:** Delete both OVN DB files and restart. `reset-dev-env.sh` does this
-automatically:
+**Fix:** Delete both OVN DB files and restart:
 
 ```bash
 sudo systemctl stop ovn-central ovn-controller
@@ -1328,21 +1421,6 @@ sudo systemctl start ovn-central ovn-controller
 # vpcd reconcile will recreate the NB topology on next startup
 ```
 
-### Chassis Name Mismatch
-
-After mulga-999, vpcd uses the OVS-managed UUID (persisted at
-`/etc/openvswitch/system-id.conf` by the `openvswitch-switch` package and
-re-applied on every boot) as the canonical chassis identity. The chassis name
-across OVS `external_ids:system-id`, OVN SB `Chassis.name`, and NB
-`Gateway_Chassis.chassis_name` is always the same value, so the mismatch
-class that this section previously documented can no longer recur.
-
-`ReconcileFromKV` runs a `reconcileGatewayChassis` pre-step on every vpcd
-startup that deletes any stale `Gateway_Chassis` row left over from earlier
-installs (where `setup-ovn.sh` fabricated `chassis-$(hostname -s)`) and
-re-binds every gateway LRP against the live SBDB chassis. Pre-mulga-999
-brokenness recovers automatically on the first vpcd restart after upgrade —
-no manual `ovn-nbctl destroy` required.
 
 ### WAN NIC Not Enslaved to a Bridge
 

docs/containers/ecr/README.md

@@ -16,7 +16,7 @@ sections:
 resources:
   - title: "Spinifex Repository"
     url: "https://github.com/mulgadc/spinifex"
-  - title: "Terraform AWS Provider — ECR"
+  - title: "Terraform AWS Provider"
     url: "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecr_repository"
   - title: "OCI Distribution Spec"
     url: "https://github.com/opencontainers/distribution-spec"

docs/containers/ecs/README.md

@@ -62,11 +62,32 @@ A **task definition** describes one or more containers (image, CPU/memory, ports
 ECS v1 is deliberately minimal. Keep these in mind:
 
 - **No service discovery.** `serviceRegistries` / Cloud Map integration is not implemented — reach a service through its load balancer, not a DNS name.
-- **`awslogs` log driver is discarded.** Container stdout/stderr is not shipped to CloudWatch Logs; the log configuration is accepted but dropped.
-- **No rolling deployments.** The reconciler keeps the desired count running but ignores `deploymentConfiguration` (`minimumHealthyPercent` / `maximumPercent`); an `UpdateService` to a new revision replaces tasks without a health-gated rollout or circuit-breaker.
+- **Only the `json-file` log driver is collected.** Container stdout/stderr is captured host-side on the container instance (see [Logging](#logging)); it is not shipped to CloudWatch Logs. Any other driver (e.g. `awslogs`) is accepted for parity but its logs are discarded — `RegisterTaskDefinition` logs a warning naming the container so the drop is not silent.
 - **No capacity providers or managed scaling.** Capacity is the static total of your registered instances; there is no scale-out/in or ASG binding — you manage the instance count.
-- **No distinct execution role.** ECR pulls use the instance role directly; `executionRoleArn` is not honoured separately.
-- **`secrets[]` are silently dropped**, and tags set via `TagResource` are not persisted. Health-check settings beyond the target group default are not forwarded.
+- **`secrets[]` are rejected.** A task definition that declares container `secrets[]` fails `RegisterTaskDefinition` with `InvalidParameterException` rather than running without the secrets it expects. Tags set via `TagResource` are not persisted, and health-check settings beyond the target group default are not forwarded.
+
+### Logging
+
+Spinifex honours the host-side **`json-file`** log driver (the containerd default). A container's stdout/stderr is written on its container instance and retrievable there — no CloudWatch Logs path exists.
+
+To read a task's logs, find the container instance running it (`aws ecs describe-tasks` → `containerInstanceArn` → the EC2 instance), then on that host inspect the containerd task output. Containers are named `{taskId}-{containerName}` and carry `mulga.ecs.*` labels:
+
+```bash
+# On the container instance:
+ctr -n default containers ls                 # find {taskId}-{containerName}
+ctr -n default tasks ls
+journalctl -u containerd | grep <taskId>     # container stdout/stderr via the host journal
+```
+
+A task definition may still declare `awslogs` (or any other driver) for AWS compatibility — Spinifex accepts it, warns at registration that the driver is not implemented, and falls back to the host-side `json-file` behaviour.
+
+### Deployments
+
+An `UpdateService` to a new task-definition revision performs a **health-gated rolling update** honouring `deploymentConfiguration`: `minimumHealthyPercent` keeps that fraction of the desired count running while `maximumPercent` bounds how many extra tasks launch during the roll. Enabling the **deployment circuit breaker** fails a rollout whose tasks repeatedly fail to start, and — with `rollback` set — automatically reverts to the last-good task definition.
+
+### Execution role
+
+If a task definition sets `executionRoleArn`, the agent assumes that role to authorise ECR image pulls (instead of the container-instance role). When it is unset, pulls fall back to the instance role.
 
 ## Prerequisites
 
@@ -267,4 +288,4 @@ tofu apply -var 'gateway_url=https://<host-lan-ip>:9999'
 
 **Container cannot assume its task role.** Confirm the task definition sets `taskRoleArn` and the role is trusted by `ecs-tasks.amazonaws.com`. The agent injects `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`; a container that overrides this variable, or an SDK too old to honour it, will not pick up the credentials.
 
-**Expected a feature that is missing.** ECS v1 omits service discovery, `awslogs` shipping, rolling deployments, capacity providers, and a distinct execution role. See the Limitations in the Overview.
+**Expected a feature that is missing.** ECS v1 omits service discovery, CloudWatch Logs (`awslogs`) shipping, and capacity providers / managed scaling. See the Limitations in the Overview.

docs/identity/iam-groups/README.md

@@ -0,0 +1,222 @@
+---
+title: "IAM Groups"
+description: "Organise IAM users into groups and manage permissions once for the whole team."
+category: "Identity"
+tags:
+  - iam
+  - groups
+  - users
+  - policies
+  - security
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS IAM Groups Documentation"
+    url: "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_groups.html"
+---
+
+# IAM Groups
+
+> Organise IAM users into groups and manage permissions once for the whole team.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Groups](#groups)
+- [Membership](#membership)
+- [Group Policies](#group-policies)
+- [Putting It All Together](#putting-it-all-together)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+A group is a collection of IAM users. Policies attached to a group apply to every member, so instead of attaching the same policy to each developer individually, you attach it once to a `developers` group and manage membership.
+
+Members inherit both the group's **attached managed policies** and its **inline policies**, combined with any policies on the user itself. Policy evaluation is the same as everywhere else: an explicit `Deny` in any applicable policy wins, and anything not allowed is denied.
+
+Groups cannot be nested, and a group is not a principal — it cannot sign requests or own access keys. Like all IAM resources, groups are scoped to the account that creates them.
+
+## Prerequisites
+
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
+## Instructions
+
+## Groups
+
+### Create a Group
+
+```bash
+aws iam create-group --group-name developers
+```
+
+```json
+{
+  "Group": {
+    "Path": "/",
+    "GroupName": "developers",
+    "GroupId": "AGPA1A2B3C4D5E6F7890",
+    "Arn": "arn:aws:iam::000000000001:group/developers",
+    "CreateDate": "2026-07-03T10:00:00Z"
+  }
+}
+```
+
+Use `--path` to organise groups hierarchically:
+
+```bash
+aws iam create-group --group-name ops --path /teams/
+```
+
+### Get and List Groups
+
+`get-group` returns the group and its members:
+
+```bash
+aws iam get-group --group-name developers
+aws iam list-groups
+aws iam list-groups --path-prefix /teams/
+```
+
+### Delete a Group
+
+A group must have no members and no policies before it can be deleted — see [Cleaning up a group](#deleteconflict-when-deleting-a-group).
+
+```bash
+aws iam delete-group --group-name developers
+```
+
+## Membership
+
+Add and remove users:
+
+```bash
+aws iam add-user-to-group --group-name developers --user-name carol
+aws iam remove-user-from-group --group-name developers --user-name carol
+```
+
+List a group's members (the `Users` array of `get-group`):
+
+```bash
+aws iam get-group --group-name developers --query 'Users[].UserName'
+```
+
+List the groups a user belongs to:
+
+```bash
+aws iam list-groups-for-user --user-name carol --query 'Groups[].GroupName'
+```
+
+> **Group membership blocks user deletion.** `delete-user` returns `DeleteConflict` while the user is still in any group — remove them from all groups first.
+
+## Group Policies
+
+### Attach a Managed Policy
+
+```bash
+aws iam attach-group-policy --group-name developers \
+  --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
+
+aws iam list-attached-group-policies --group-name developers
+
+aws iam detach-group-policy --group-name developers \
+  --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
+```
+
+See [IAM Users and Policies](/docs/iam-users-and-policies) for creating managed policies and the policy document format.
+
+### Inline Group Policies
+
+Inline policies live inside the group and are deleted with it:
+
+```bash
+aws iam put-group-policy --group-name developers \
+  --policy-name keypair-mgmt \
+  --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["ec2:CreateKeyPair","ec2:DeleteKeyPair"],"Resource":"*"}]}'
+
+aws iam list-group-policies --group-name developers
+aws iam get-group-policy --group-name developers --policy-name keypair-mgmt
+aws iam delete-group-policy --group-name developers --policy-name keypair-mgmt
+```
+
+Members are authorised by the union of the group's attached and inline policies. A user in `developers` with the policies above can call `ec2:DescribeInstances` (via the attached `EC2ReadOnly`) and `ec2:CreateKeyPair` (via the inline policy), while anything not granted — say `ec2:RunInstances` — is denied.
+
+## Putting It All Together
+
+Onboard a team with shared permissions:
+
+```bash
+# 1. Create the group and grant it permissions
+aws iam create-group --group-name developers
+aws iam attach-group-policy --group-name developers \
+  --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
+
+# 2. Create users and add them to the group
+aws iam create-user --user-name carol
+aws iam add-user-to-group --group-name developers --user-name carol
+
+# 3. Create access keys and configure profiles as usual
+aws iam create-access-key --user-name carol
+
+# 4. Verify: carol can describe instances via group policy, nothing more
+AWS_PROFILE=spinifex-carol aws ec2 describe-instances
+AWS_PROFILE=spinifex-carol aws ec2 run-instances --image-id ami-123 --instance-type t3.micro
+# ^ AccessDenied — not granted by the group
+```
+
+To change the whole team's permissions later, edit the group's policies once — no per-user changes needed.
+
+## Troubleshooting
+
+### DeleteConflict When Deleting a Group
+
+The group still has members, attached policies, or inline policies. Empty it first:
+
+```bash
+# Members
+aws iam get-group --group-name developers --query 'Users[].UserName'
+aws iam remove-user-from-group --group-name developers --user-name carol
+
+# Attached policies
+aws iam list-attached-group-policies --group-name developers
+aws iam detach-group-policy --group-name developers --policy-arn arn:aws:iam::000000000001:policy/EC2ReadOnly
+
+# Inline policies
+aws iam list-group-policies --group-name developers
+aws iam delete-group-policy --group-name developers --policy-name keypair-mgmt
+
+aws iam delete-group --group-name developers
+```
+
+### DeleteConflict When Deleting a User
+
+Group membership counts as a subordinate entity, alongside access keys and attached policies. Find and leave the user's groups:
+
+```bash
+aws iam list-groups-for-user --user-name carol
+aws iam remove-user-from-group --group-name developers --user-name carol
+```
+
+### NoSuchEntity
+
+The group or user referenced does not exist — both `get-group` on a missing group and `add-user-to-group` with a missing user return this. Check spelling with `list-groups` / `list-users`.
+
+### Member Still Denied After Attaching a Group Policy
+
+Check the policy landed on the right group and the user is actually a member:
+
+```bash
+aws iam list-groups-for-user --user-name carol
+aws iam list-attached-group-policies --group-name developers
+aws iam list-group-policies --group-name developers
+```
+
+Remember an explicit `Deny` in any policy that applies to the user — their own or any of their groups' — overrides the `Allow`.

docs/identity/iam-roles-and-instance-profiles/README.md

@@ -0,0 +1,376 @@
+---
+title: "IAM Roles and Instance Profiles"
+description: "Create IAM roles with trust policies, wrap them in instance profiles, and grant EC2 instances credentials without static keys."
+category: "Identity"
+tags:
+  - iam
+  - roles
+  - instance profiles
+  - trust policies
+  - credentials
+  - security
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS IAM Roles Documentation"
+    url: "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html"
+  - title: "AWS Instance Profiles Documentation"
+    url: "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html"
+---
+
+# IAM Roles and Instance Profiles
+
+> Create IAM roles with trust policies, wrap them in instance profiles, and grant EC2 instances credentials without static keys.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Roles](#roles)
+- [Granting Permissions to a Role](#granting-permissions-to-a-role)
+- [Instance Profiles](#instance-profiles)
+- [Launching an Instance with a Role](#launching-an-instance-with-a-role)
+- [Managing Profile Associations at Runtime](#managing-profile-associations-at-runtime)
+- [Assuming a Role](#assuming-a-role)
+- [Cleaning Up](#cleaning-up)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+A role is an IAM identity with permissions but no long-lived credentials. Instead of an access key pair, a role has a **trust policy** declaring who may assume it; whoever assumes the role receives short-lived, auto-rotating STS credentials.
+
+An **instance profile** is the container that binds a role to an EC2 instance. An instance launched with a profile gets the role's credentials delivered through [IMDS](/docs/imds) — no static keys baked into the image, no `~/.aws/credentials` inside the guest.
+
+The full arc is: create a role → attach permissions → wrap it in an instance profile → launch an instance with the profile → the guest picks up credentials automatically. Each instance profile holds exactly **one role** (matching AWS).
+
+Like all IAM resources, roles and instance profiles are scoped to the account that creates them.
+
+## Prerequisites
+
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
+## Instructions
+
+## Roles
+
+### Create a Role
+
+Every role needs a trust policy. For a role that EC2 instances will use, trust the `ec2.amazonaws.com` service principal:
+
+```bash
+cat > /tmp/ec2-trust.json << 'EOF'
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com" },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+EOF
+
+aws iam create-role --role-name app-server \
+  --assume-role-policy-document file:///tmp/ec2-trust.json \
+  --description "Role for app servers"
+```
+
+```json
+{
+  "Role": {
+    "Path": "/",
+    "RoleName": "app-server",
+    "RoleId": "AROA1A2B3C4D5E6F7890",
+    "Arn": "arn:aws:iam::000000000001:role/app-server",
+    "CreateDate": "2026-07-03T10:00:00Z",
+    "AssumeRolePolicyDocument": {
+      "Version": "2012-10-17",
+      "Statement": [
+        {
+          "Effect": "Allow",
+          "Principal": { "Service": "ec2.amazonaws.com" },
+          "Action": "sts:AssumeRole"
+        }
+      ]
+    },
+    "Description": "Role for app servers",
+    "MaxSessionDuration": 3600
+  }
+}
+```
+
+To let a user (rather than EC2) assume the role, trust an `AWS` principal instead:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "AWS": "arn:aws:iam::000000000001:user/admin" },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+```
+
+**Trust policy validation:** Spinifex rejects `NotPrincipal`, `NotAction`, empty `Principal` blocks, and empty-string `Action` elements at write time with `MalformedPolicyDocument`. `Condition` blocks are rejected except the `StringEquals` form used for web identity federation.
+
+### Get, List, and Update Roles
+
+```bash
+aws iam get-role --role-name app-server
+aws iam list-roles
+aws iam list-roles --path-prefix /services/
+```
+
+Update the description or the maximum STS session duration (3600 seconds by default):
+
+```bash
+aws iam update-role --role-name app-server \
+  --description "Updated" --max-session-duration 7200
+```
+
+Replace the trust policy on an existing role:
+
+```bash
+aws iam update-assume-role-policy --role-name app-server \
+  --policy-document file:///tmp/ec2-trust.json
+```
+
+## Granting Permissions to a Role
+
+A freshly created role can do nothing. Grant permissions the same two ways as users: attach managed policies, or embed inline policies.
+
+### Attach a Managed Policy
+
+```bash
+aws iam attach-role-policy --role-name app-server \
+  --policy-arn arn:aws:iam::000000000001:policy/S3ReadOnly
+
+aws iam list-attached-role-policies --role-name app-server
+aws iam detach-role-policy --role-name app-server \
+  --policy-arn arn:aws:iam::000000000001:policy/S3ReadOnly
+```
+
+See [IAM Users and Policies](/docs/iam-users-and-policies) for creating managed policies and the policy document format.
+
+### Inline Role Policies
+
+Inline policies live inside the role and are deleted with it:
+
+```bash
+aws iam put-role-policy --role-name app-server \
+  --policy-name ec2-describe \
+  --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["ec2:DescribeInstances"],"Resource":"*"}]}'
+
+aws iam list-role-policies --role-name app-server
+aws iam get-role-policy --role-name app-server --policy-name ec2-describe
+aws iam delete-role-policy --role-name app-server --policy-name ec2-describe
+```
+
+## Instance Profiles
+
+Create a profile and add the role to it:
+
+```bash
+aws iam create-instance-profile --instance-profile-name app-server-profile
+aws iam add-role-to-instance-profile \
+  --instance-profile-name app-server-profile --role-name app-server
+```
+
+Inspect it — the `Roles` array shows the bound role:
+
+```bash
+aws iam get-instance-profile --instance-profile-name app-server-profile
+aws iam list-instance-profiles
+aws iam list-instance-profiles-for-role --role-name app-server
+```
+
+A profile holds at most one role; adding a second returns `LimitExceeded`. To swap roles, remove the current one first:
+
+```bash
+aws iam remove-role-from-instance-profile \
+  --instance-profile-name app-server-profile --role-name app-server
+```
+
+## Launching an Instance with a Role
+
+Pass the profile by name (or ARN) at launch:
+
+```bash
+aws ec2 run-instances \
+  --image-id ami-0dd52c90440ff4150 \
+  --instance-type t3.micro \
+  --subnet-id subnet-a0e5fc381376d82a1 \
+  --iam-instance-profile Name=app-server-profile
+```
+
+The instance description shows the association:
+
+```bash
+aws ec2 describe-instances --instance-ids i-d2e09ff7de71b6341 \
+  --query 'Reservations[0].Instances[0].IamInstanceProfile'
+```
+
+```json
+{
+  "Arn": "arn:aws:iam::000000000001:instance-profile/app-server-profile",
+  "Id": "AIPA1A2B3C4D5E6F7890"
+}
+```
+
+Inside the guest, the AWS CLI and SDKs pick up the role's credentials from IMDS with no configuration — see [IMDS](/docs/imds) for fetching them manually, rotation timing, and limits.
+
+## Managing Profile Associations at Runtime
+
+Profiles can be attached to, removed from, or swapped on a **running** instance — no relaunch needed.
+
+List associations:
+
+```bash
+aws ec2 describe-iam-instance-profile-associations \
+  --filters Name=instance-id,Values=i-d2e09ff7de71b6341
+```
+
+```json
+{
+  "IamInstanceProfileAssociations": [
+    {
+      "AssociationId": "iip-assoc-9d7a617c605e337cd",
+      "InstanceId": "i-d2e09ff7de71b6341",
+      "IamInstanceProfile": {
+        "Arn": "arn:aws:iam::000000000001:instance-profile/app-server-profile"
+      },
+      "State": "associated"
+    }
+  ]
+}
+```
+
+Attach a profile to an instance launched without one:
+
+```bash
+aws ec2 associate-iam-instance-profile \
+  --instance-id i-d2e09ff7de71b6341 \
+  --iam-instance-profile Name=app-server-profile
+```
+
+Swap to a different profile (returns a new association ID):
+
+```bash
+aws ec2 replace-iam-instance-profile-association \
+  --association-id iip-assoc-9d7a617c605e337cd \
+  --iam-instance-profile Name=other-profile
+```
+
+Remove the profile — the guest's `iam/` metadata subtree starts returning 404:
+
+```bash
+aws ec2 disassociate-iam-instance-profile \
+  --association-id iip-assoc-9d7a617c605e337cd
+```
+
+## Assuming a Role
+
+Users and services can assume a role directly with STS, provided the trust policy allows their principal:
+
+```bash
+aws sts assume-role \
+  --role-arn arn:aws:iam::000000000001:role/deploy \
+  --role-session-name release-42
+```
+
+```json
+{
+  "Credentials": {
+    "AccessKeyId": "ASIA1A2B3C4D5E6F7890",
+    "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+    "SessionToken": "IQoJb3JpZ2luX2VjE...",
+    "Expiration": "2026-07-03T11:00:00Z"
+  },
+  "AssumedRoleUser": {
+    "AssumedRoleId": "AROA1A2B3C4D5E6F7890:release-42",
+    "Arn": "arn:aws:sts::000000000001:assumed-role/deploy/release-42"
+  }
+}
+```
+
+The returned `ASIA`-prefixed credentials are temporary, valid up to the role's `MaxSessionDuration`. See [STS and Temporary Credentials](/docs/sts) for session durations, using the credentials, and the other STS flows.
+
+## Cleaning Up
+
+Deletion order matters — a role must be empty and unreferenced before it can be deleted:
+
+```bash
+# 1. Remove the role from any instance profiles
+aws iam remove-role-from-instance-profile \
+  --instance-profile-name app-server-profile --role-name app-server
+aws iam delete-instance-profile --instance-profile-name app-server-profile
+
+# 2. Delete inline policies and detach managed policies
+aws iam delete-role-policy --role-name app-server --policy-name ec2-describe
+aws iam detach-role-policy --role-name app-server \
+  --policy-arn arn:aws:iam::000000000001:policy/S3ReadOnly
+
+# 3. Delete the role
+aws iam delete-role --role-name app-server
+```
+
+## Troubleshooting
+
+### MalformedPolicyDocument When Creating a Role
+
+The trust policy is invalid. In addition to the JSON checks that apply to all policies, trust policies must not use `NotPrincipal`, `NotAction`, an empty `Principal` block, or an empty-string `Action`. `Condition` blocks are rejected except `StringEquals` conditions used with web identity federation.
+
+### DeleteConflict When Deleting a Role
+
+The role still has attached managed policies, inline policies, or is bound to an instance profile. Remove all three first:
+
+```bash
+aws iam list-attached-role-policies --role-name app-server
+aws iam list-role-policies --role-name app-server
+aws iam list-instance-profiles-for-role --role-name app-server
+```
+
+### DeleteConflict When Deleting an Instance Profile
+
+The profile still contains a role:
+
+```bash
+aws iam remove-role-from-instance-profile \
+  --instance-profile-name app-server-profile --role-name app-server
+aws iam delete-instance-profile --instance-profile-name app-server-profile
+```
+
+### LimitExceeded When Adding a Role to a Profile
+
+The profile already holds a role — the limit is one per profile. Remove the existing role first, or create a second profile.
+
+### InvalidIamInstanceProfile.NotFound at Launch
+
+The profile name or ARN passed to `run-instances` does not exist in your account. Check the spelling:
+
+```bash
+aws iam list-instance-profiles
+```
+
+### Instance Has No Credentials in IMDS
+
+If the `iam/` metadata subtree returns 404, the instance has no associated profile. Attach one without relaunching:
+
+```bash
+aws ec2 associate-iam-instance-profile \
+  --instance-id i-d2e09ff7de71b6341 \
+  --iam-instance-profile Name=app-server-profile
+```
+
+If the profile is associated but contains no role, the `security-credentials/` listing is empty; add the role and credentials appear on the next request.

docs/identity/iam-users-and-policies/README.md

@@ -1,7 +1,7 @@
 ---
 title: "IAM Users and Policies"
 description: "Create IAM users, manage access keys, and control permissions with policies."
-category: "Administration"
+category: "Identity"
 tags:
   - iam
   - users
@@ -30,31 +30,29 @@ resources:
 - [Policies](#policies)
 - [Attaching Policies](#attaching-policies)
 - [Putting It All Together](#putting-it-all-together)
-- [Command Reference](#command-reference)
 - [Troubleshooting](#troubleshooting)
 
 ---
 
 ## Overview
 
-Spinifex implements AWS-compatible IAM with 16 operations covering user management, access key lifecycle, policy CRUD, and policy attachment. All IAM resources are scoped to the account that creates them — users in one account cannot see or modify resources in another.
+Spinifex implements AWS-compatible IAM covering user management, access key lifecycle, policy CRUD, and policy attachment. Inline user policies (`put-user-policy`), user and policy tagging, [groups](/docs/iam-groups), and [roles and instance profiles](/docs/iam-roles-and-instance-profiles) are also supported. All IAM resources are scoped to the account that creates them — users in one account cannot see or modify resources in another.
 
 When you create an account with `spx admin account create`, Spinifex bootstraps a root user with an `AdministratorAccess` policy and writes the credentials to `~/.aws/credentials`. From there, you use the standard AWS CLI to manage additional users and permissions.
 
 **How authentication works:** Every AWS CLI request is signed with SigV4 using an access key pair. The gateway verifies the signature, resolves the caller's account, and evaluates attached policies before routing the request. The root user (account `000000000000`) bypasses policy evaluation entirely.
 
-## Instructions
-
 ## Prerequisites
 
-- A running Spinifex cluster with the AWS gateway started
-- An account created via `spx admin account create` (see [Creating Accounts](/docs/creating-accounts))
-- AWS CLI configured with the account's profile:
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:
 
 ```bash
-export AWS_PROFILE=spinifex-myteam
+export AWS_PROFILE=spinifex
 ```
 
+## Instructions
+
 ## Users
 
 ### Create a User
@@ -143,7 +141,17 @@ aws iam create-access-key --user-name alice
 
 > **The secret access key is only shown once.** Save it immediately. If lost, delete the key and create a new one.
 
-Add the credentials to `~/.aws/credentials` under a new profile and set `AWS_PROFILE` to use them.
+Configure a profile for the new key. A Spinifex profile also needs the gateway endpoint and CA bundle (copy the values from your existing `spinifex-*` profile in `~/.aws/config`):
+
+```bash
+aws configure set aws_access_key_id AKIA1A2B3C4D5E6F7890ABCD --profile spinifex-alice
+aws configure set aws_secret_access_key wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY --profile spinifex-alice
+aws configure set region ap-southeast-2 --profile spinifex-alice
+aws configure set endpoint_url https://localhost:9999 --profile spinifex-alice
+aws configure set ca_bundle /var/lib/spinifex/config/ca.pem --profile spinifex-alice
+```
+
+Then set `AWS_PROFILE=spinifex-alice` to use it.
 
 ### List Access Keys
 
@@ -159,7 +167,9 @@ Create a second key, update your configuration, then delete the old one:
 # 1. Create new key (while old key still works)
 aws iam create-access-key --user-name alice
 
-# 2. Update ~/.aws/credentials with the new key
+# 2. Update the profile with the new key
+aws configure set aws_access_key_id AKIA_NEW_KEY_ID --profile spinifex-alice
+aws configure set aws_secret_access_key NEW_SECRET --profile spinifex-alice
 
 # 3. Verify the new key works
 AWS_PROFILE=spinifex-alice aws sts get-caller-identity
@@ -433,9 +443,13 @@ aws iam create-policy \
 aws iam attach-user-policy --user-name dev-carol \
   --policy-arn arn:aws:iam::000000000001:policy/DeveloperAccess
 
-# 5. Configure the developer's AWS CLI
-aws configure --profile spinifex-carol
-# Enter the access key and secret from step 2
+# 5. Configure the developer's AWS CLI profile with the key from step 2.
+#    endpoint_url and ca_bundle: copy from an existing spinifex-* profile in ~/.aws/config
+aws configure set aws_access_key_id AKIA_CAROL_KEY_ID --profile spinifex-carol
+aws configure set aws_secret_access_key CAROL_SECRET --profile spinifex-carol
+aws configure set region ap-southeast-2 --profile spinifex-carol
+aws configure set endpoint_url https://localhost:9999 --profile spinifex-carol
+aws configure set ca_bundle /var/lib/spinifex/config/ca.pem --profile spinifex-carol
 
 # 6. Verify
 AWS_PROFILE=spinifex-carol aws ec2 describe-instances
@@ -443,35 +457,14 @@ AWS_PROFILE=spinifex-carol aws ec2 terminate-instances --instance-ids i-123
 # ^ This will be denied by the NoTerminate statement
 ```
 
-## Command Reference
-
-| Command                       | Key Parameters                               | Notes                             |
-| ----------------------------- | -------------------------------------------- | --------------------------------- |
-| `create-user`                 | `--user-name`, `--path`                      | Path default: `/`                 |
-| `get-user`                    | `--user-name`                                |                                   |
-| `list-users`                  | `--path-prefix`                              |                                   |
-| `delete-user`                 | `--user-name`                                | Must have no keys or policies     |
-| `create-access-key`           | `--user-name`                                | Max 2 per user; secret shown once |
-| `list-access-keys`            | `--user-name`                                |                                   |
-| `delete-access-key`           | `--user-name`, `--access-key-id`             |                                   |
-| `update-access-key`           | `--user-name`, `--access-key-id`, `--status` | `Active` or `Inactive`            |
-| `create-policy`               | `--policy-name`, `--policy-document`         | Max 6144 bytes                    |
-| `get-policy`                  | `--policy-arn`                               | Metadata only                     |
-| `get-policy-version`          | `--policy-arn`, `--version-id`               | Use `v1`; includes document       |
-| `list-policies`               |                                              |                                   |
-| `delete-policy`               | `--policy-arn`                               | Must not be attached              |
-| `attach-user-policy`          | `--user-name`, `--policy-arn`                | Idempotent                        |
-| `detach-user-policy`          | `--user-name`, `--policy-arn`                |                                   |
-| `list-attached-user-policies` | `--user-name`                                |                                   |
-
 ## Troubleshooting
 
 ### AccessDenied on IAM Commands
 
-The calling user must have IAM permissions. Attach a policy with `iam:*` actions, or use the root account profile:
+The calling user must have IAM permissions. Attach a policy with `iam:*` actions, or use the account's admin profile:
 
 ```bash
-export AWS_PROFILE=spinifex-myteam
+export AWS_PROFILE=spinifex
 aws iam list-users
 ```
 
@@ -494,7 +487,7 @@ If the key was deleted, create a new one.
 
 ### DeleteConflict When Deleting a User
 
-The user still has access keys or attached policies. Remove them first:
+The user still has access keys, attached policies, or [group memberships](/docs/iam-groups). Remove them first:
 
 ```bash
 # Check for access keys
@@ -502,6 +495,9 @@ aws iam list-access-keys --user-name alice
 
 # Check for attached policies
 aws iam list-attached-user-policies --user-name alice
+
+# Check for group memberships
+aws iam list-groups-for-user --user-name alice
 ```
 
 ### DeleteConflict When Deleting a Policy

docs/identity/imds/README.md

@@ -0,0 +1,349 @@
+---
+title: "IMDS (Instance Metadata Service)"
+description: "Access instance metadata, user data, and IAM role credentials from inside a guest VM with IMDSv2."
+category: "Identity"
+tags:
+  - imds
+  - metadata
+  - instance identity
+  - iam roles
+  - credentials
+  - security
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS IMDS Documentation"
+    url: "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html"
+  - title: "AWS IMDSv2 Deep Dive"
+    url: "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html"
+---
+
+# Instance Metadata Service (IMDS)
+
+> Access instance metadata, user data, and IAM role credentials from inside a guest VM with IMDSv2.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Using IMDS from a Guest](#using-imds-from-a-guest)
+- [Metadata Paths](#metadata-paths)
+- [User Data](#user-data)
+- [IAM Role Credentials](#iam-role-credentials)
+- [Instance Identity Document](#instance-identity-document)
+- [Metadata Options](#metadata-options)
+- [How It Works](#how-it-works)
+- [Limits and Defaults](#limits-and-defaults)
+- [Command Reference](#command-reference)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+Every running guest VM can reach the Instance Metadata Service at `http://169.254.169.254`, exactly as on EC2. There is no in-VM agent to install and no in-guest route configuration — DHCP and fully static guests reach it identically.
+
+IMDS answers questions a workload asks about itself: instance ID, instance type, private and public IPs, hostname, availability zone, security groups, the launch SSH key, user data, and — when the instance has an IAM instance profile — short-lived, auto-rotating role credentials.
+
+**Spinifex is IMDSv2-only.** Every read requires a session token obtained with a `PUT` request. A tokenless (IMDSv1-style) `GET` returns `401 Unauthorized` with an empty body.
+
+Requests are attributed to an instance by the network interface they arrive on and tokens are bound to that interface, so one instance can never read another's metadata or replay its tokens.
+
+## Prerequisites
+
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- A running instance to query from
+- For IAM role credentials: an instance launched with `--iam-instance-profile`
+
+## Instructions
+
+## Using IMDS from a Guest
+
+All commands in this section run **inside the guest VM**.
+
+### Get a Session Token
+
+Request a token with a TTL between 1 and 21600 seconds (6 hours):
+
+```bash
+TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" \
+  -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+```
+
+The token endpoint only accepts `PUT` — a `GET` returns `405 Method Not Allowed`. A missing or out-of-range TTL header returns `400 Bad Request`.
+
+### Read Metadata
+
+Send the token back in the `X-aws-ec2-metadata-token` header on every read:
+
+```bash
+curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
+  http://169.254.169.254/latest/meta-data/instance-id
+```
+
+```
+i-0a1b2c3d4e5f67890
+```
+
+Directory paths (ending in `/`) return a newline-separated listing of children:
+
+```bash
+curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
+  http://169.254.169.254/latest/meta-data/
+```
+
+### One-Liner Pattern
+
+For scripts, mint a short-lived token inline:
+
+```bash
+imds() {
+  local token=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" \
+    -H "X-aws-ec2-metadata-token-ttl-seconds: 60")
+  curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/$1"
+}
+
+imds meta-data/instance-id
+imds meta-data/local-ipv4
+imds meta-data/placement/availability-zone
+```
+
+## Metadata Paths
+
+`GET /` lists supported API versions (`2021-07-15`, `latest`). Any dated version segment (e.g. `/2016-09-02/...`, as probed by cloud-init) aliases to `/latest`.
+
+Commonly used paths under `/latest/meta-data/`:
+
+| Path | Returns |
+| ---- | ------- |
+| `instance-id` | Instance ID |
+| `instance-type` | Instance type (e.g. `t3.micro`) |
+| `ami-id` | Image the instance was launched from |
+| `ami-launch-index` | Launch index within the reservation (`0..n-1`) |
+| `reservation-id` | Reservation ID |
+| `instance-life-cycle` | `spot` or `on-demand` |
+| `local-ipv4` | Primary private IP |
+| `public-ipv4` | Elastic/public IP; 404 when none |
+| `public-hostname` | Mirrors `public-ipv4`; 404 when no public IP |
+| `mac` | Primary interface MAC address |
+| `hostname`, `local-hostname` | `ip-<dashed-ip>.<region>.compute.internal` |
+| `security-groups` | Security group names, one per line |
+| `placement/availability-zone` | Availability zone |
+| `placement/region` | Region (AZ with trailing letter stripped) |
+| `services/domain`, `services/partition` | `amazonaws.com` / `aws` |
+| `public-keys/0/openssh-key` | Launch key pair's SSH public key; 404 if no key pair or the key was deleted |
+| `iam/info` | Instance profile ARN and ID; 404 if no profile |
+| `iam/security-credentials/<role>` | Temporary role credentials (see below) |
+| `network/interfaces/macs/<mac>/...` | Primary interface subtree: `interface-id`, `owner-id`, `subnet-id`, `vpc-id`, `local-ipv4s`, `security-group-ids`, `subnet-ipv4-cidr-block`, `vpc-ipv4-cidr-block`, and more |
+
+The `network/interfaces/macs/` subtree covers the **primary interface only**; querying another MAC returns 404 (multi-ENI metadata is deferred).
+
+Paths that intentionally return **404**: `tags/instance/*`, `block-device-mapping/*`, `placement/{group-name,partition-number,availability-zone-id,host-id}`, `instance-action`, and `spot/{instance-action,termination-time}` (404 is the correct "no interruption scheduled" answer for spot pollers).
+
+## User Data
+
+User data supplied at launch (`run-instances --user-data`) is served at:
+
+```bash
+curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
+  http://169.254.169.254/latest/user-data
+```
+
+Returns the decoded user data, or 404 if the instance was launched without any. cloud-init consumes this automatically at first boot.
+
+## IAM Role Credentials
+
+Instances launched with an IAM instance profile get short-lived, auto-rotating credentials through IMDS — no static keys baked into the image. Creating roles and instance profiles is covered in [IAM Roles and Instance Profiles](/docs/iam-roles-and-instance-profiles).
+
+```bash
+# Discover the role name
+ROLE=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
+  http://169.254.169.254/latest/meta-data/iam/security-credentials/)
+
+# Fetch credentials
+curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
+  "http://169.254.169.254/latest/meta-data/iam/security-credentials/$ROLE"
+```
+
+```json
+{
+  "Code": "Success",
+  "LastUpdated": "2026-07-03T10:00:00Z",
+  "Type": "AWS-HMAC",
+  "AccessKeyId": "ASIA1A2B3C4D5E6F7890",
+  "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+  "Token": "IQoJb3JpZ2luX2VjE...",
+  "Expiration": "2026-07-03T11:00:00Z",
+  "AccountId": "000000000001"
+}
+```
+
+Credentials are ASIA-prefixed temporary STS credentials valid for **1 hour**, re-minted automatically **5 minutes before expiry**. The AWS SDKs and CLI pick them up with no configuration:
+
+```bash
+# Inside the guest, with no ~/.aws/credentials:
+aws sts get-caller-identity
+```
+
+If the instance has no profile, the whole `iam/` subtree returns 404 (and is omitted from the `meta-data/` listing).
+
+## Instance Identity Document
+
+An unsigned identity document (schema `2017-09-30`) is available at:
+
+```bash
+curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
+  http://169.254.169.254/latest/dynamic/instance-identity/document
+```
+
+```json
+{
+  "accountId": "000000000001",
+  "architecture": "x86_64",
+  "availabilityZone": "ap-southeast-2a",
+  "imageId": "ami-0123456789abcdef0",
+  "instanceId": "i-0a1b2c3d4e5f67890",
+  "instanceType": "t3.micro",
+  "pendingTime": "2026-07-03T09:58:12Z",
+  "privateIp": "10.0.1.15",
+  "region": "ap-southeast-2",
+  "version": "2017-09-30"
+}
+```
+
+The signed forms (`signature`, `pkcs7`, `rsa2048`) currently return 404 — they require a per-cluster signing key and land with EKS IRSA support.
+
+## Metadata Options
+
+Metadata options are managed from the **control plane** with the standard EC2 commands. Because the platform is IMDSv2-only, the only mutable knob is the PUT response hop limit.
+
+### Inspect
+
+```bash
+aws ec2 describe-instances --instance-ids i-0a1b2c3d4e5f67890 \
+  --query 'Reservations[0].Instances[0].MetadataOptions'
+```
+
+```json
+{
+  "State": "applied",
+  "HttpTokens": "required",
+  "HttpPutResponseHopLimit": 1,
+  "HttpEndpoint": "enabled",
+  "HttpProtocolIpv6": "disabled",
+  "InstanceMetadataTags": "disabled"
+}
+```
+
+### Set at Launch
+
+```bash
+aws ec2 run-instances \
+  --image-id ami-0123456789abcdef0 \
+  --instance-type t3.micro \
+  --metadata-options "HttpPutResponseHopLimit=2"
+```
+
+### Modify a Running or Stopped Instance
+
+```bash
+aws ec2 modify-instance-metadata-options \
+  --instance-id i-0a1b2c3d4e5f67890 \
+  --http-put-response-hop-limit 2
+```
+
+Raise the hop limit above the default of 1 when containers on the instance need IMDS access through an extra routing hop (e.g. containers on a bridge network fetching role credentials). Valid range is 1–64.
+
+Rejected settings:
+
+- `--http-tokens optional` → `UnsupportedOperation` (IMDSv2 cannot be relaxed)
+- `--http-endpoint disabled` → `UnsupportedOperation` (the endpoint cannot be turned off)
+- `--http-protocol-ipv6 enabled` → `UnsupportedOperation`
+- `--instance-metadata-tags enabled` → `UnsupportedOperation`
+- Hop limit outside 1–64 → `InvalidParameterValue`
+
+## How It Works
+
+Useful background for host-side troubleshooting:
+
+- The IMDS HTTP server runs inside the `spinifex-vpcd` service on each host, listening on `169.254.169.254:80` with one listener per instance's primary interface — attribution is structural, with no source-IP trust.
+- A freshly launched guest's metadata service comes up within a few seconds of launch; cloud-init's built-in retries absorb this window.
+- Session tokens and cached role credentials are not persisted. A vpcd restart drops them; SDKs and cloud-init transparently reissue. Datapath state on the host survives service restarts.
+
+## Limits and Defaults
+
+| Setting | Value |
+| ------- | ----- |
+| Token TTL | 1–21600 seconds (request-scoped, via header) |
+| Token binding | Issuing network interface; in-memory, not persisted |
+| Role credential lifetime | 3600 seconds, refreshed 5 minutes before expiry |
+| PUT response hop limit | Default 1, valid 1–64 |
+| `HttpTokens` | Always `required` (immutable) |
+| `HttpEndpoint` | Always `enabled` (immutable) |
+| Tokenless GET | `401 Unauthorized`, empty body |
+| `X-Forwarded-For` present | `403 Forbidden` |
+| Wrong method | `405 Method Not Allowed` |
+| Unknown/unsupported path | `404 Not Found` |
+
+## Troubleshooting
+
+### 401 Unauthorized on Every Read
+
+The request is missing a valid token. Common causes:
+
+- No `X-aws-ec2-metadata-token` header — IMDSv1-style access is not supported; obtain a token first.
+- The token expired — reissue with a fresh `PUT /latest/api/token`.
+- The token was issued to a different instance/interface — tokens are interface-bound and rejected elsewhere, identically to unknown tokens.
+- The token endpoint itself never 401s; if the `PUT` fails, check for a `400` (bad TTL header) instead.
+
+### 403 Forbidden
+
+The request carried an `X-Forwarded-For` header. IMDS rejects proxied requests outright; call it directly from the workload, not through a forward proxy.
+
+### Software in a Container Cannot Reach IMDS
+
+The default hop limit of 1 means the PUT response dies at the first routed hop, e.g. a Docker bridge network. Raise it from the control plane:
+
+```bash
+aws ec2 modify-instance-metadata-options \
+  --instance-id i-0a1b2c3d4e5f67890 \
+  --http-put-response-hop-limit 2
+```
+
+### UnsupportedOperation When Setting Metadata Options
+
+You attempted to relax IMDSv2 enforcement (`--http-tokens optional`) or disable the endpoint (`--http-endpoint disabled`). Neither is supported — the platform posture is fixed. Only the hop limit can be changed.
+
+### IAM Credentials Return 404, an Empty List, or "Code": "Failed"
+
+A 404 or empty list means the instance has no IAM instance profile, or the profile has no role. Verify from the control plane:
+
+```bash
+aws ec2 describe-instances --instance-ids i-0a1b2c3d4e5f67890 \
+  --query 'Reservations[0].Instances[0].IamInstanceProfile'
+```
+
+A credential body with `"Code": "Failed"` means the backend could not mint credentials (for example, the role was deleted after launch). Check the role still exists and the vpcd logs on the host for `IMDS: AssumeRoleForInstance failed`. See the IAM Roles and Instance Profiles guide for setting up profiles.
+
+### Connection Refused / Timeout to 169.254.169.254
+
+Metadata is served per-interface by `spinifex-vpcd` on the host. In order:
+
+1. Immediately after launch, wait a few seconds — the listener converges within one 15-second reconcile tick, and cloud-init retries through it.
+2. On the host, confirm vpcd is running and serving:
+
+```bash
+systemctl status spinifex-vpcd
+journalctl -u spinifex-vpcd | grep 'IMDS:'
+```
+
+Look for `IMDS: tap responder serving` (listener bound) and `IMDS: issued IMDSv2 token` (proof guest packets are traversing the datapath). A bound responder with no token issuance points at the host datapath rather than the HTTP service:
+
+```bash
+ovs-vsctl list-ports br-imds          # per-instance endpoint + patch ports
+ovs-vsctl list-ports br-int | grep imi-
+```
+
+### cloud-init Did Not Apply SSH Key or User Data
+
+cloud-init sources both from IMDS at first boot. Check `public-keys/0/openssh-key` and `user-data` respond from inside the guest (see [Using IMDS from a Guest](#using-imds-from-a-guest)), and review `/var/log/cloud-init.log` in the guest. A 404 on `public-keys/` means the instance was launched without `--key-name` or the key pair was since deleted.

docs/identity/sts/README.md

@@ -0,0 +1,346 @@
+---
+title: "STS and Temporary Credentials"
+description: "Mint short-lived credentials with STS — assume roles, issue session tokens, and federate Kubernetes workloads with OIDC."
+category: "Identity"
+tags:
+  - sts
+  - iam
+  - temporary credentials
+  - assume role
+  - oidc
+  - security
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "AWS STS Documentation"
+    url: "https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html"
+  - title: "AWS Temporary Security Credentials"
+    url: "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html"
+---
+
+# STS and Temporary Credentials
+
+> Mint short-lived credentials with STS — assume roles, issue session tokens, and federate Kubernetes workloads with OIDC.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Checking Your Identity](#checking-your-identity)
+- [Assuming a Role](#assuming-a-role)
+- [Using Temporary Credentials](#using-temporary-credentials)
+- [Session Tokens for Your Own User](#session-tokens-for-your-own-user)
+- [Web Identity Federation (IRSA)](#web-identity-federation-irsa)
+- [Trust Policy Validation](#trust-policy-validation)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+STS (Security Token Service) issues **temporary credentials**: an `ASIA`-prefixed access key, a secret key, and a session token, all of which expire together at a fixed time. Nothing to rotate, nothing to revoke — the credentials simply stop working.
+
+Spinifex supports the three main ways to obtain them:
+
+- **`assume-role`** — exchange your IAM user credentials for a [role's](/docs/iam-roles-and-instance-profiles) permissions, gated by the role's trust policy.
+- **`get-session-token`** — get a time-boxed copy of your own user's permissions.
+- **`assume-role-with-web-identity`** — exchange an OIDC ID token (a Kubernetes ServiceAccount token) for role credentials, with no IAM credentials at all.
+
+EC2 instances get role credentials a fourth way, automatically through [IMDS](/docs/imds) — no STS call needed in the guest.
+
+## Prerequisites
+
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:
+
+```bash
+export AWS_PROFILE=spinifex
+```
+
+## Instructions
+
+## Checking Your Identity
+
+`get-caller-identity` returns whoever signed the request — it needs no permissions and never fails for authorisation reasons, which makes it the standard "which credentials am I using?" check:
+
+```bash
+aws sts get-caller-identity
+```
+
+```json
+{
+    "UserId": "AIDA1A2B3C4D5E6F7890",
+    "Account": "000000000001",
+    "Arn": "arn:aws:iam::000000000001:user/admin"
+}
+```
+
+Run with temporary role credentials, the ARN switches to the assumed-role form — see [Using Temporary Credentials](#using-temporary-credentials).
+
+## Assuming a Role
+
+`assume-role` works when the target role's trust policy allows your principal. Create a role that trusts an IAM user (see [IAM Roles and Instance Profiles](/docs/iam-roles-and-instance-profiles) for the full role lifecycle):
+
+```bash
+cat > /tmp/deploy-trust.json << 'EOF'
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "AWS": "arn:aws:iam::000000000001:user/admin" },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+EOF
+
+aws iam create-role --role-name deploy \
+  --assume-role-policy-document file:///tmp/deploy-trust.json
+
+aws iam put-role-policy --role-name deploy --policy-name ec2-read \
+  --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["ec2:Describe*"],"Resource":"*"}]}'
+```
+
+Then assume it:
+
+```bash
+aws sts assume-role \
+  --role-arn arn:aws:iam::000000000001:role/deploy \
+  --role-session-name release-42
+```
+
+```json
+{
+    "Credentials": {
+        "AccessKeyId": "ASIA1A2B3C4D5E6F7890",
+        "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+        "SessionToken": "IQoJb3JpZ2luX2VjE...",
+        "Expiration": "2026-07-03T11:00:00Z"
+    },
+    "AssumedRoleUser": {
+        "AssumedRoleId": "AROA1A2B3C4D5E6F7890:release-42",
+        "Arn": "arn:aws:sts::000000000001:assumed-role/deploy/release-42"
+    },
+    "PackedPolicySize": 0
+}
+```
+
+The session name tags every session so audit logs can tell *who* used the role; it appears in the assumed-role ARN and the `UserId`.
+
+If the trust policy does not allow your principal, `assume-role` fails with `AccessDenied` — no matter what IAM permissions you hold.
+
+### Session Duration
+
+`--duration-seconds` must be between **900** (15 minutes) and the smaller of the role's `MaxSessionDuration` and **43200** (12 hours); the default is **3600** (1 hour). A value outside that window is rejected with `ValidationError`, so to run longer sessions, raise the role's ceiling first:
+
+```bash
+# 7200 > MaxSessionDuration (3600) → ValidationError
+aws sts assume-role --role-arn arn:aws:iam::000000000001:role/deploy \
+  --role-session-name long-build --duration-seconds 7200
+
+aws iam update-role --role-name deploy --max-session-duration 43200
+
+aws sts assume-role --role-arn arn:aws:iam::000000000001:role/deploy \
+  --role-session-name long-build --duration-seconds 7200
+```
+
+### Flags That Differ from AWS
+
+- `--external-id` and `--source-identity` are accepted and logged but **not enforced** — trust policies cannot carry the `Condition` blocks that would check them (see [Trust Policy Validation](#trust-policy-validation)). Do not rely on an external ID as a security boundary.
+- `--serial-number` / `--token-code` (MFA) are not supported and return `InvalidParameterValue`.
+- `--policy` / `--policy-arns` (session policies) are not supported and return `PackedPolicyTooLarge`.
+
+## Using Temporary Credentials
+
+Temporary credentials are used exactly like access keys, plus a third value — the session token. Export all three as environment variables (which take precedence over any profile):
+
+```bash
+export AWS_ACCESS_KEY_ID=ASIA1A2B3C4D5E6F7890
+export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+export AWS_SESSION_TOKEN=IQoJb3JpZ2luX2VjE...
+export AWS_ENDPOINT_URL=https://localhost:9999
+export AWS_CA_BUNDLE=/var/lib/spinifex/config/ca.pem
+export AWS_DEFAULT_REGION=ap-southeast-2
+```
+
+Or store them in a named profile — the same shape as a user profile with `aws_session_token` added:
+
+```bash
+aws configure set aws_access_key_id ASIA1A2B3C4D5E6F7890 --profile spinifex-deploy
+aws configure set aws_secret_access_key wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY --profile spinifex-deploy
+aws configure set aws_session_token IQoJb3JpZ2luX2VjE... --profile spinifex-deploy
+aws configure set region ap-southeast-2 --profile spinifex-deploy
+aws configure set endpoint_url https://localhost:9999 --profile spinifex-deploy
+aws configure set ca_bundle /var/lib/spinifex/config/ca.pem --profile spinifex-deploy
+```
+
+Requests are then authorised as the assumed role, not your user:
+
+```bash
+aws sts get-caller-identity
+# "Arn": "arn:aws:sts::000000000001:assumed-role/deploy/release-42"
+
+aws ec2 describe-instances     # allowed by the role's ec2-read policy
+aws iam list-users             # AccessDenied — the role grants ec2:Describe* only
+```
+
+When the expiration time passes, every call fails until you fetch fresh credentials — there is no refresh; run `assume-role` again.
+
+## Session Tokens for Your Own User
+
+`get-session-token` issues temporary credentials with your **own user's** permissions — no role, no trust policy. Useful for handing a build script credentials that expire on their own instead of your long-lived access key:
+
+```bash
+aws sts get-session-token --duration-seconds 3600
+```
+
+```json
+{
+    "Credentials": {
+        "AccessKeyId": "ASIA1A2B3C4D5E6F7890",
+        "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+        "SessionToken": "IQoJb3JpZ2luX2VjE...",
+        "Expiration": "2026-07-03T11:00:00Z"
+    }
+}
+```
+
+Duration runs from **900** to **129600** seconds (36 hours), defaulting to **43200** (12 hours). Unlike `assume-role`, out-of-range values are **clamped, not rejected** — asking for 200000 seconds quietly returns a 36-hour session. Use the credentials exactly as in [Using Temporary Credentials](#using-temporary-credentials); `get-caller-identity` keeps reporting your user ARN, since no role is involved.
+
+MFA (`--serial-number` / `--token-code`) is not supported and returns `InvalidParameterValue`.
+
+## Web Identity Federation (IRSA)
+
+`assume-role-with-web-identity` exchanges an OIDC ID token for role credentials — the mechanism behind **IRSA** (IAM Roles for Service Accounts), where a Kubernetes pod's ServiceAccount token becomes its cloud identity. The call is anonymous: the JWT is the identity, so no IAM credentials sign the request.
+
+In Spinifex the token issuer is an [EKS](/docs/eks) cluster. Each cluster publishes a signing-key set (JWKS) under an issuer URL of the form `https://{host}/oidc/eks/{region}/{account-id}/{cluster-name}`, and STS verifies tokens against it directly — no external identity provider is contacted.
+
+### Register the OIDC Provider
+
+The role account must register the issuer as an IAM OIDC provider before any token from it is accepted:
+
+```bash
+aws iam create-open-id-connect-provider \
+  --url https://spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod \
+  --client-id-list sts.amazonaws.com \
+  --thumbprint-list 9e99a48a9960b14926bb7f3b02e22da2b0ab7280
+```
+
+```json
+{
+    "OpenIDConnectProviderArn": "arn:aws:iam::000000000001:oidc-provider/spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod",
+    "Tags": []
+}
+```
+
+Manage providers with the usual verbs — `get-open-id-connect-provider` and `list-open-id-connect-provider-tags` take the ARN, and tags work like every other IAM resource:
+
+```bash
+aws iam list-open-id-connect-providers
+aws iam get-open-id-connect-provider \
+  --open-id-connect-provider-arn arn:aws:iam::000000000001:oidc-provider/spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod
+aws iam delete-open-id-connect-provider \
+  --open-id-connect-provider-arn arn:aws:iam::000000000001:oidc-provider/spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod
+```
+
+### The IRSA Trust Policy
+
+The role's trust policy names the provider as a `Federated` principal and pins the token's `sub` (the ServiceAccount) and `aud` claims with `StringEquals` — the one place a `Condition` block is allowed:
+
+```bash
+cat > /tmp/irsa-trust.json << 'EOF'
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws:iam::000000000001:oidc-provider/spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod"
+      },
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod:sub": "system:serviceaccount:default:app",
+          "spinifex.example.com/oidc/eks/ap-southeast-2/000000000001/prod:aud": "sts.amazonaws.com"
+        }
+      }
+    }
+  ]
+}
+EOF
+
+aws iam create-role --role-name irsa-app \
+  --assume-role-policy-document file:///tmp/irsa-trust.json
+```
+
+The condition keys are the issuer URL (scheme stripped) followed by `:sub` or `:aud`. Only these two keys and only `StringEquals` are accepted — anything wider (`StringLike`, wildcards, other keys) is rejected at `create-role` with `MalformedPolicyDocument` rather than silently over-granting.
+
+### Exchange the Token
+
+```bash
+aws sts assume-role-with-web-identity \
+  --role-arn arn:aws:iam::000000000001:role/irsa-app \
+  --role-session-name pod-app-7f9c4 \
+  --web-identity-token "$(cat /var/run/secrets/eks.amazonaws.com/serviceaccount/token)"
+```
+
+The response has the same `Credentials` block as `assume-role`, plus the verified `SubjectFromWebIdentityToken`, `Provider`, and `Audience`. Duration runs 900–43200 seconds (default 3600), independent of the role's `MaxSessionDuration`.
+
+For a token to be accepted it must be an ES256-signed JWT whose `iss` matches a registered OIDC provider in the role's account, whose `aud` contains `sts.amazonaws.com`, whose signature verifies against the cluster's published JWKS, and which has not expired — any failure returns `InvalidIdentityToken`. In-cluster, none of this is manual: pods with an annotated ServiceAccount get the token mounted and the AWS SDK performs the exchange automatically.
+
+## Trust Policy Validation
+
+Spinifex validates trust policies (`AssumeRolePolicyDocument`) more strictly than AWS, rejecting at write time with `MalformedPolicyDocument`:
+
+- `Condition` blocks anywhere **except** the `StringEquals` IRSA form on `sts:AssumeRoleWithWebIdentity` described above. This is why `--external-id` cannot be enforced.
+- `NotPrincipal` and `NotAction` elements.
+- Empty `Principal` blocks and empty-string `Action` elements.
+
+Everything the validator accepts is evaluated at assume time; explicit `Deny` statements win, and a policy that matches no statement denies.
+
+## Troubleshooting
+
+### AccessDenied When Assuming a Role
+
+The role's trust policy does not allow your principal, or the role does not exist — a missing role is deliberately indistinguishable from a denied one, matching AWS. Check who you are, that the role exists, and what it trusts:
+
+```bash
+aws sts get-caller-identity
+aws iam list-roles --query 'Roles[].Arn'
+aws iam get-role --role-name deploy \
+  --query 'Role.AssumeRolePolicyDocument'
+```
+
+For web identity, `AccessDenied` also covers a trust-policy `Condition` that doesn't match the token's `sub`/`aud` claims.
+
+### ValidationError on Duration
+
+The requested `--duration-seconds` is outside 900–min(`MaxSessionDuration`, 43200). Check and raise the role's ceiling:
+
+```bash
+aws iam get-role --role-name deploy --query 'Role.MaxSessionDuration'
+aws iam update-role --role-name deploy --max-session-duration 43200
+```
+
+Note `get-session-token` never returns this — it clamps instead.
+
+### InvalidParameterValue
+
+You passed an MFA flag (`--serial-number` / `--token-code`) — MFA is not supported. `--tags` and `--transitive-tag-keys` on `assume-role` return the same error.
+
+### PackedPolicyTooLarge
+
+You passed `--policy` or `--policy-arns`. Session policies are not supported — scope the role's own permissions instead.
+
+### InvalidIdentityToken
+
+The web identity token failed verification: malformed JWT, wrong signature algorithm (only ES256 is accepted), expired, `aud` missing `sts.amazonaws.com`, issuer not registered as an OIDC provider in the role's account, or the cluster's JWKS is unavailable. Confirm the provider is registered:
+
+```bash
+aws iam list-open-id-connect-providers
+```
+
+### Credentials Suddenly Rejected
+
+Temporary credentials expired. Check the `Expiration` from the original response and mint a fresh set — sessions cannot be renewed or extended.

docs/install-vm-eval/README.md

@@ -97,8 +97,8 @@ iface br-wan inet dhcp
 
 Walkthrough of changes:
 
-* Adds eno3 into the bridge
-* eno3 becomes a slave interface
+* Adds enp0s1 into the bridge
+* enp0s1 becomes a slave interface
 * All traffic goes through br-wan
 * Spinifex EC2 (VMs) in a public subnet, obtain an IP address from the WAN upstream
 

docs/install/install-airgapped/README.md

@@ -1,7 +1,7 @@
 ---
 title: "Air-Gapped Install"
 description: "Deploy Spinifex in environments without internet connectivity. Covers using a pre-built release tarball, mirrored APT packages, and locally-staged cloud images."
-category: "Getting Started"
+category: "Install"
 tags:
   - install
   - air-gapped

docs/install/install-multi-node/README.md

@@ -1,7 +1,7 @@
 ---
 title: "Multi-Node Install"
 description: "Deploy Spinifex across multiple servers to create an availability zone with high availability, data durability, and fault tolerance."
-category: "Getting Started"
+category: "Install"
 tags:
   - install
   - multi node
@@ -45,7 +45,7 @@ A Spinifex cluster distributes services across multiple servers for high availab
 >
 > Before running the installer on any server, that server's WAN interface **must** already be enslaved to a Linux bridge named `br-wan`. The host IP, default route, and DHCP must all live on the bridge — not on the bare NIC.
 >
-> The bootstrap installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in the next release.
+> The bootstrap installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in a future release.
 >
 > **Verify on every node before continuing:**
 >

docs/install/install-source/README.md

@@ -1,7 +1,7 @@
 ---
 title: "Source Install"
 description: "Build Spinifex from source for development, custom builds, or contributing."
-category: "Getting Started"
+category: "Install"
 tags:
   - install
   - source
@@ -29,12 +29,10 @@ resources:
 
 This guide builds Spinifex from source. For production deployments, the [binary installer](/docs/install) is recommended.
 
-**Requirements:**
+**Supported Operating Systems:**
 
-- Ubuntu 26.04 / Debian 13
-- Go 1.26.4+
-- GCC, make, pkg-config
-- QEMU/KVM, OVN/Open vSwitch, AWS CLI v2, dhcpcd-base
+- Ubuntu 26.04
+- Debian 13
 
 ## Instructions
 
@@ -47,13 +45,15 @@ sudo make -C spinifex quickinstall
 export PATH=$PATH:/usr/local/go/bin
 ```
 
-## Step 2. Clone and Build
+## Step 2. Clone Sibling Repositories
 
 ```bash
 cd spinifex
 ./scripts/clone-deps.sh
 ```
 
+This clones Predastore (S3) and Viperblock (EBS) alongside Spinifex.
+
 ## Step 3. Development Initialisation
 
 ```bash

docs/install/install-usb/README.md

@@ -1,7 +1,7 @@
 ---
 title: "Bootable USB Install"
 description: "Install Spinifex on bare-metal hardware by flashing the Spinifex ISO to a USB drive and booting the target device from it."
-category: "Getting Started"
+category: "Install"
 sections:
   - overview
   - troubleshooting

docs/install/install/README.md

@@ -1,7 +1,7 @@
 ---
 title: "Single-Node Install"
 description: "Install Spinifex on a single server using the binary installer."
-category: "Getting Started"
+category: "Install"
 tags:
   - install
   - single node
@@ -55,7 +55,7 @@ This guide installs Spinifex on a single server using the binary installer. For
 >
 > Before running the installer, the host's WAN interface **must** already be enslaved to a Linux bridge named `br-wan`. The host IP, default route, and DHCP must all live on the bridge — not on the bare NIC.
 >
-> The bootstrap installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in the next release.
+> The bootstrap installer does **not** create this bridge for you yet. Running it on a host whose default route is still on a bare NIC will leave the install in a non-working state. Auto-provisioning of `br-wan` will land in a future release.
 >
 > **Verify before continuing:**
 >

docs/migration/hybrid-sync/README.md

@@ -31,7 +31,8 @@ Spinifex's hybrid mode enables bidirectional data synchronization between local
 
 ## Prerequisites
 
-Ensure the AWS profile is set:
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:
 
 ```bash
 export AWS_PROFILE=spinifex

docs/migration/moving-aws-workload/README.md

@@ -31,15 +31,16 @@ resources:
 
 Spinifex provides drop-in compatibility with AWS APIs, making it possible to migrate existing workloads with minimal changes.
 
-**Supported Services:** VPC, EC2, EBS, S3, IAM
+**Supported Services:** EC2, VPC, EBS, S3, IAM, STS, ELBv2 (ALB/NLB), ACM, ECR, ECS, EKS
 
-**Compatible Tools:** AWS CLI, AWS SDKs, Terraform, any S3-compatible client
+**Compatible Tools:** AWS CLI, AWS SDKs, Terraform, kubectl (via `aws eks get-token`), any S3-compatible client
 
 ## Instructions
 
 ## Prerequisites
 
-Ensure the AWS profile is set:
+- A running Spinifex cluster (see [Setting Up Your Cluster](/docs/setting-up-your-cluster))
+- AWS CLI configured with the `spinifex` profile:
 
 ```bash
 export AWS_PROFILE=spinifex
@@ -61,13 +62,21 @@ provider "aws" {
   secret_key = "your-spinifex-secret-key"
 
   endpoints {
-    ec2 = "https://localhost:9999"
-    s3  = "https://localhost:8443"
+    ec2                = "https://localhost:9999"
+    iam                = "https://localhost:9999"
+    sts                = "https://localhost:9999"
+    elbv2              = "https://localhost:9999"
+    acm                = "https://localhost:9999"
+    ecr                = "https://localhost:9999"
+    ecs                = "https://localhost:9999"
+    eks                = "https://localhost:9999"
+    s3                 = "https://localhost:8443"
   }
 
   skip_credentials_validation = true
   skip_metadata_api_check     = true
   skip_requesting_account_id  = true
+  skip_region_validation      = true
 }
 ```
 
@@ -81,12 +90,13 @@ aws s3 sync s3://local-bucket/ s3://cloud-bucket/ --source-region spinifex --reg
 
 ## Terraform Provider Errors
 
-Ensure all three skip flags are set in your provider configuration:
+Ensure all four skip flags are set in your provider configuration:
 
 ```hcl
 skip_credentials_validation = true
 skip_metadata_api_check     = true
 skip_requesting_account_id  = true
+skip_region_validation      = true
 ```
 
 Without these, Terraform will try to validate credentials and metadata against real AWS endpoints.

docs/security/malware-protection/README.md

@@ -266,7 +266,7 @@ aideinit
 mv /var/lib/aide/aide.db.new /var/lib/aide/aide.db
 ```
 
-Daily check — a systemd timer invoking `aide --check`. Non-zero exit must alert. Re-baseline after every Spinifex upgrade (`make reinstall`, package upgrade, config change) and record the new hash.
+Daily check — a systemd timer invoking `aide --check`. Non-zero exit must alert. Re-baseline after every Spinifex upgrade, package upgrade, or config change and record the new hash.
 
 ### 4.3 Wazuh (centralised AV + FIM + alerting)
 

docs/security/media-sanitization/README.md

@@ -77,7 +77,7 @@ Any medium in any of these roles on a Spinifex node has held, or may have held,
 | Viperblock chunk cache / local backing | NVMe/SATA SSD or HDD | Encrypted volume chunks. |
 | Predastore object store | NVMe/SATA SSD or HDD | All S3 object data — AMIs, snapshots, user-uploaded artifacts, IAM state files, tenant data. |
 | NATS JetStream disk | NVMe/SATA SSD | IAM NATS token, cluster metadata, pending job state, NATS KV entries (including wrapped DEKs). |
-| Removable master-key media | USB flash token, HSM, smart card | Master encryption key (Phase 4 USB-mount path). Destroying this sanitizes every DEK-encrypted volume cluster-wide. |
+| Removable master-key media | USB flash token, HSM, smart card | Master encryption key (USB-mount path). Destroying this sanitizes every DEK-encrypted volume cluster-wide. |
 | Backup media | LTO tape, removable HDD/SSD, off-site cloud backup | Any of the above, point-in-time. |
 | BMC / iDRAC / iLO storage | Embedded flash | Console recordings, SEL logs, BMC credentials, cached operator certificates. Sanitize via the BMC "Reset to defaults" / "Erase user data" command before disposal. |
 | Switch / router config storage | Embedded flash | Cluster VLAN, management IPs, ACLs. Operator network kit, out of Spinifex scope but noted for completeness. |
@@ -193,7 +193,7 @@ Third-party destruction vendors must provide a certificate of destruction listin
 When retiring a whole node:
 
 1. **Drain** — migrate or terminate all instances hosted on the node; confirm Predastore and Viperblock roles are reassigned if the node held one. Volume DEKs of terminated instances are already cryptographically erased per [§3.1](#31-cryptographic-erase-via-dek-deletion).
-2. **Deregister** — `bd`-linked change ticket; the operator removes the node from the cluster (`spx admin node remove`) so NATS routes, OVN chassis, and predastore distributed membership drop it.
+2. **Deregister** — raise a change ticket; the operator removes the node from the cluster (`spx admin node remove`) so NATS routes, OVN chassis, and predastore distributed membership drop it.
 3. **Stop services and unmount** — `systemctl stop 'spinifex-*'`, `nbdkit` sessions, OVN agent. Confirm `/run/spinifex/nbd/` is empty.
 4. **Destroy on-disk keys** — see [§5](#5-key-destruction). This is the single most important step: sanitization of volume data reduces to sanitization of keys once data is encrypted.
 5. **Sanitize each drive** by media type per [§4.1](#41-nvme-ssd) – [§4.3](#43-hdd). Every drive bay is sanitized, including any unpopulated cache or WAL devices.
@@ -211,7 +211,7 @@ Cryptographic erase relies on the key being unrecoverable. A drive that previous
 | Cluster CA private key (`/etc/spinifex/ca.key`) | Leader node system disk | Same treatment. Loss of the CA key does not sanitize data, but prevents impersonation of the cluster identity post-disposal. |
 | Per-node TLS private key (`/etc/spinifex/server.key`) | Each node system disk | `shred -u` before drive sanitization. |
 | Wrapped DEKs in NATS KV | `/var/lib/spinifex/nats/` on NATS-carrying nodes | Deleted via the `DeleteVolume` API; the KV record is compacted. Sanitization of the NATS JetStream disk per [§4](#4-whole-drive-and-node-decommissioning--before-disposal-mpl1-383-a) ensures no recoverable copies. |
-| Removable master-key media (Phase 4 USB) | USB flash token | Physically destroy the token when the cluster is decommissioned; alternatively retain inside a secured location as evidence of cluster-wide cryptographic erase. Do not reuse for another cluster without full sanitization (see [§6](#6-removable-and-backup-media)). |
+| Removable master-key media (USB) | USB flash token | Physically destroy the token when the cluster is decommissioned; alternatively retain inside a secured location as evidence of cluster-wide cryptographic erase. Do not reuse for another cluster without full sanitization (see [§6](#6-removable-and-backup-media)). |
 | Backup-copy keys | Any off-site or escrow copy | Destroy simultaneously with the primary. A surviving backup defeats the cryptographic erase. |
 
 If even one copy of the master key survives and the encrypted drives are recoverable, cryptographic erase has **not** been achieved. Track every key copy in the device register ([Physical Security Guide §5](/docs/security/physical-security-guide#5-manage-physical-access-devices-pel1-3105)).

docs/security/network-connections/README.md

@@ -88,12 +88,14 @@ The inventory in [§1](#1-inbound-listeners)–[§2](#2-outbound-connections) sa
 | 8443 | spinifex-predastore | HTTPS | Cluster | S3-compatible object storage (AMIs, snapshots, user objects) | AWS SigV4 + TLS |
 | 6660–6662 | predastore (Raft) | TCP | Cluster | Metadata consensus (3 nodes) | Cluster network only |
 | 9991–9993 | predastore (data shards) | TCP | Cluster | Erasure-coded data shard transport | Cluster network only |
-| 6641 | OVN Northbound DB | OVSDB/TCP | Cluster | Logical network topology consumed by vpcd | Cluster network only; TLS planned |
-| 6642 | OVN Southbound DB | OVSDB/TCP | Cluster | Chassis / port / MAC binding state | Cluster network only; TLS planned |
+| 6641 | OVN Northbound DB (client) | OVSDB/TCP | Cluster | Logical network topology consumed by vpcd | Cluster network only; TLS planned |
+| 6642 | OVN Southbound DB (client) | OVSDB/TCP | Cluster | Chassis / port / MAC binding state | Cluster network only; TLS planned |
+| 6643 | OVN Northbound DB (RAFT) | OVSDB/TCP | Cluster | NB database RAFT replication between the 3 quorum nodes | Cluster network only; TLS planned |
+| 6644 | OVN Southbound DB (RAFT) | OVSDB/TCP | Cluster | SB database RAFT replication between the 3 quorum nodes | Cluster network only; TLS planned |
 | 8222 | spinifex-nats (monitoring) | HTTP | Localhost | `varz`/`subsz` metrics consumed by the daemon | Loopback only |
 | socket / dynamic TCP | nbdkit (Viperblock) | NBD | Host-local / cluster | Block device transport for guest EBS volumes | Unix socket by default; TCP only in remote/DPU mode |
 
-¹ **Formation port lifecycle.** 4432 opens during `spx admin init` / `spx admin join` while a bootstrap token is outstanding and closes once the cluster is formed (token TTL default 30 min, `--token-ttl`). The server presents an ephemeral self-signed cert that pre-dates trust bootstrap, so the joining node dials with `InsecureSkipVerify` — the only production dial that does. Authenticity rests on the operator supplying the leader address out-of-band plus possession of the bearer token. Document in the security plan so reviewers do not flag 4432 as a persistent open port.
+¹ **Formation port lifecycle.** 4432 opens during `spx admin init` / `spx admin join` while a bootstrap token is outstanding and closes once the cluster is formed (token TTL default 30 min, `--token-ttl`). The server presents an ephemeral self-signed cert that pre-dates trust bootstrap, so the joining node does not verify the certificate chain for this single dial. Authenticity rests on the operator supplying the leader address out-of-band plus possession of the bearer token. Document in the security plan so reviewers do not flag 4432 as a persistent open port.
 
 **Development-only listeners.** When `dev_networking=true`, QEMU opens arbitrary host TCP ports for SSH port-forwarding into guest VMs. Production installs (the `/etc/spinifex` layout) do not enable this; it must not appear on compliance nodes.
 
@@ -126,7 +128,8 @@ Control-plane and data-plane traffic between Spinifex nodes, for completeness an
 | Predastore S3 | 8443 | TLS + AWS SigV4 | Cross-node object reads/writes |
 | Predastore Raft | 6660–6662 | Cluster network only | Metadata consensus |
 | Predastore shards | 9991–9993 | Cluster network only | Erasure-coded data shards |
-| OVN NB/SB | 6641 / 6642 | Cluster network only (TLS planned) | Network control plane |
+| OVN NB/SB (client) | 6641 / 6642 | Cluster network only (TLS planned) | Network control plane; vpcd and ovn-controller dial the quorum |
+| OVN NB/SB (RAFT) | 6643 / 6644 | Cluster network only (TLS planned) | NB/SB database replication across the 3 quorum nodes |
 | OVN tunnels (Geneve) | UDP 6081 | None | Tenant traffic overlay between chassis, inside the cluster subnet |
 
 Nodes **must** sit on a network segment that is not routed to tenant/guest VMs or to the internet. Predastore Raft/shards and OVN DBs are cluster-internal and must not be reachable from anywhere else.
@@ -140,7 +143,7 @@ Default external surface is three listeners — **9999** (AWS API), **3000** (UI
 tcp dport { 22, 9999, 3000 } accept
 
 # Cluster-only: replace 10.0.1.0/24 with your cluster CIDR
-ip saddr 10.0.1.0/24 tcp dport { 4222, 4248, 8443, 6641, 6642, 6660-6662, 9991-9993 } accept
+ip saddr 10.0.1.0/24 tcp dport { 4222, 4248, 8443, 6641-6644, 6660-6662, 9991-9993 } accept
 ip saddr 10.0.1.0/24 udp dport 6081 accept
 
 # Default deny
@@ -162,7 +165,7 @@ Every listener and outbound destination is controlled by one of these files. Cha
 | OVN packages (`ovn-central`, `ovn-host`) | `ovn-nb-db`, `ovn-sb-db` (via `ovs-vsctl set open_vswitch …`) | OVN DB bind addresses. |
 | Spinifex UI service | Built-in defaults: `host = "0.0.0.0"`, `port = 3000`. No `spinifex.toml` block today. | UI listener. |
 | `spx admin init` / `spx admin join` | `--port`, `--token-ttl`, `--no-telemetry` (or `SPX_NO_TELEMETRY=1`) | Formation port, token TTL, telemetry opt-out. |
-| `utils/images.go` `AvailableImages` | Image catalogue URLs | Outbound HTTPS destinations for image downloads. |
+| Image catalogue (built-in) | Fixed URLs listed in [§2](#2-outbound-connections); not operator-configurable | Outbound HTTPS destinations for image downloads. |
 
 ## 6. Operator Checklist
 
@@ -172,6 +175,6 @@ Every listener and outbound destination is controlled by one of these files. Cha
 - Formation port 4432 is closed on nodes not actively running a bootstrap token.
 - Outbound HTTPS restricted to the [§2](#2-outbound-connections) image-catalogue hosts, or replaced with air-gapped import.
 - Install telemetry (`install.mulgadc.com`) is either permitted and recorded in the security plan, or disabled via `SPX_NO_TELEMETRY=1` / `--no-telemetry`.
-- OVN NB/SB (6641/6642) exposure limited to the cluster subnet pending the L2 TLS work.
+- OVN NB/SB client and RAFT ports (6641–6644) exposure limited to the cluster subnet pending the L2 TLS work.
 - SSH (22) configured to operator-managed keys only; password auth disabled in `sshd_config`.
 - Periodic review (at least annually, and after any topology change) confirms this inventory still matches the deployed configuration.

docs/service-interfaces.yaml

@@ -332,6 +332,12 @@ suites:
   # e2e-bm:
   #   path: spinifex/tests/e2e/baremetal
   #   covers: [awsgw, spinifex_daemon, vpcd, viperblockd, predastore]
+  #
+  # e2e-natuplink activates once CI can provision an external_mode=nat
+  # node (setup-ovn.sh --nat-uplink + spx admin init --external-mode=nat).
+  # e2e-natuplink:
+  #   path: spinifex/tests/e2e/natuplink
+  #   covers: [awsgw, spinifex_daemon, vpcd, viperblockd, predastore]
 
 
 # Special-case rules consumed by the selector (Bead 4). Hard-coded in

docs/terraform-workbooks/eks-gitops-argocd/README.md

@@ -77,7 +77,7 @@ Terraform manages the cluster, the addons, the ACM cert, and the HTTPS Ingress;
 
 - The `argocd` and `aws-ebs-csi-driver` bundles must be baked into the `eks-node` AMI — `describe-addon` returns `no baked bundle` otherwise.
 - The EBS-CSI default StorageClass uses `provisioner: ebs.csi.aws.com`; a PVC against it provisions a **Viperblock-backed EBS volume**.
-- **Verify dynamic provisioning on a live cluster.** EBS-CSI dynamic provisioning depends on the running Spinifex EBS/Viperblock data plane and may not be available on every build yet. If the PVC stays `Pending`, the app still serves (the counter falls back to in-memory) — confirm the volume binds on a stack where the driver is healthy.
+- If the PVC stays `Pending`, the app still serves (the counter falls back to in-memory) — see [Troubleshooting](#troubleshooting).
 - Leave addon versions unset (the AWS provider and the catalog disagree on the version-string format).
 - The Argo CD `Application` is a `kubernetes_manifest`, so the `argoproj.io` CRDs must exist before `apply` — apply the parent and let the addon reach `ACTIVE` first.
 
@@ -256,7 +256,7 @@ kubectl describe pvc <name>
 kubectl get storageclass
 ```
 
-Confirm the `aws-ebs-csi-driver` addon is `ACTIVE` and the default StorageClass exists. EBS-CSI dynamic provisioning depends on the live Spinifex EBS/Viperblock data plane — verify on a stack where the driver is healthy. The app still serves with an in-memory counter if the volume never binds.
+Confirm the `aws-ebs-csi-driver` addon is `ACTIVE` and the default StorageClass exists. The app still serves with an in-memory counter if the volume never binds.
 
 ### The Ingress Has No Address
 

docs/terraform-workbooks/eks-https-ingress/README.md

@@ -176,8 +176,6 @@ aws eks describe-addon --cluster-name eks-https --addon-name aws-load-balancer-c
 
 Confirm the addon is `ACTIVE` and the cluster carries `spinifex.io/managed-ingress = "false"` (otherwise the built-in ingress competes).
 
-> **Verify on a live cluster.** The LBC reconcile path (ALB creation, target registration, listener wiring) exercises the running Spinifex EKS data plane end to end. Stand this up on a live stack to confirm the ALB serves the page; the manifests here are the AWS-parity shape of that flow.
-
 ### HTTPS Loads but Returns 502 / 504
 
 The ALB reached the workers but the backend didn't answer. Check the demo pods and the NodePort SG rule:

go.mod

@@ -1,9 +1,13 @@
 module github.com/mulgadc/spinifex
 
-go 1.26.4
+go 1.26.5
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
+	github.com/aws/aws-sdk-go-v2 v1.42.1
+	github.com/aws/aws-sdk-go-v2/config v1.32.27
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.26
+	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
@@ -13,10 +17,10 @@ require (
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/uuid v1.6.0
 	github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2
-	github.com/klauspost/cpuid/v2 v2.3.0
-	github.com/mulgadc/predastore v1.10.0
-	github.com/mulgadc/viperblock v1.10.0
-	github.com/nats-io/nats-server/v2 v2.14.2
+	github.com/klauspost/cpuid/v2 v2.4.0
+	github.com/mulgadc/predastore v1.11.0
+	github.com/mulgadc/viperblock v1.11.0
+	github.com/nats-io/nats-server/v2 v2.14.3
 	github.com/nats-io/nats.go v1.52.0
 	github.com/opencontainers/runtime-spec v1.3.0
 	github.com/ovn-kubernetes/libovsdb v0.8.1
@@ -25,12 +29,21 @@ require (
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0
+	go.opentelemetry.io/contrib/instrumentation/runtime v0.69.0
+	go.opentelemetry.io/otel v1.44.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0
+	go.opentelemetry.io/otel/metric v1.44.0
+	go.opentelemetry.io/otel/sdk v1.44.0
+	go.opentelemetry.io/otel/sdk/metric v1.44.0
+	go.opentelemetry.io/otel/trace v1.44.0
 	go.uber.org/goleak v1.3.0
+	go.yaml.in/yaml/v3 v3.0.4
 	golang.org/x/crypto v0.53.0
 	golang.org/x/sync v0.21.0
 	golang.org/x/sys v0.46.0
 	gopkg.in/ini.v1 v1.67.3
-	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -42,13 +55,22 @@ require (
 	github.com/antithesishq/antithesis-sdk-go v0.7.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.42.0 // indirect
-	github.com/aws/smithy-go v1.27.1 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.31.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.43.5 // indirect
+	github.com/aws/smithy-go v1.27.3 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/buraksezer/consistent v0.10.0 // indirect
-	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cenkalti/hub v1.0.2 // indirect
 	github.com/cenkalti/rpc2 v1.0.5 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
@@ -91,6 +113,7 @@ require (
 	github.com/google/flatbuffers v25.12.19+incompatible // indirect
 	github.com/google/go-tpm v0.9.8 // indirect
 	github.com/gookit/color v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-metrics v0.5.4 // indirect
@@ -150,12 +173,9 @@ require (
 	go.etcd.io/bbolt v1.4.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
-	go.opentelemetry.io/otel v1.44.0 // indirect
-	go.opentelemetry.io/otel/metric v1.44.0 // indirect
-	go.opentelemetry.io/otel/trace v1.44.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
-	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/mod v0.36.0 // indirect
 	golang.org/x/net v0.56.0 // indirect
 	golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 // indirect
@@ -164,9 +184,11 @@ require (
 	golang.org/x/time v0.15.0 // indirect
 	golang.org/x/tools v0.45.0 // indirect
 	golang.org/x/vuln v1.1.4 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260618152121-87f3d3e198d3 // indirect
 	google.golang.org/grpc v1.81.1 // indirect
 	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 // go.opencensus.io -> grpc@v1.33.2 drags in the pre-split monolithic genproto,

go.sum

@@ -522,66 +522,78 @@ github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZw
 github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
 github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo=
 github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
-github.com/aws/aws-sdk-go-v2 v1.42.0 h1:XvXMJTkFQtpBKIWZnmr9ZEOc2InWM2yldjXEJ/bymhA=
-github.com/aws/aws-sdk-go-v2 v1.42.0/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 h1:p1BBrg/Hhp6uK7zpejeI8QFXHJeC/mynzi04Sl03k9g=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13/go.mod h1:8cIfkE9MDhkRZGpQ22aV6/lkYeYSozpz16Smrs5x4Ls=
+github.com/aws/aws-sdk-go-v2 v1.42.1 h1:9eOTgu1z/dVtYpNZ3/8/XbbaX0x/BqE3HUzAzs6K0ek=
+github.com/aws/aws-sdk-go-v2 v1.42.1/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E=
 github.com/aws/aws-sdk-go-v2/config v1.18.14/go.mod h1:0pI6JQBHKwd0JnwAZS3VCapLKMO++UL2BOkWwyyzTnA=
 github.com/aws/aws-sdk-go-v2/config v1.27.4/go.mod h1:zq2FFXK3A416kiukwpsd+rD4ny6JC7QSkp4QdN1Mp2g=
 github.com/aws/aws-sdk-go-v2/config v1.29.12/go.mod h1:xse1YTjmORlb/6fhkWi8qJh3cvZi4JoVNhc+NbJt4kI=
+github.com/aws/aws-sdk-go-v2/config v1.32.27 h1:SJwJ9Q4kM7v5QVSYYyXj3znRr6lNyZEhSgAXmXXcVbI=
+github.com/aws/aws-sdk-go-v2/config v1.32.27/go.mod h1:uBfrzTRedDmB2u+b6+UlaKJy2O6VSH5un2jP24t/KvQ=
 github.com/aws/aws-sdk-go-v2/credentials v1.13.14/go.mod h1:85ckagDuzdIOnZRwws1eLKnymJs3ZM1QwVC1XcuNGOY=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.4/go.mod h1:+30tpwrkOgvkJL1rUZuRLoxcJwtI/OkeBLYnHxJtVe0=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.65/go.mod h1:4zyjAuGOdikpNYiSGpsGz8hLGmUzlY8pc8r9QQ/RXYQ=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.24 h1:2hQqYCV9yqyePQ9o6dCrZc/zO8U3TwPr9mIKlZnPu/I=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.24/go.mod h1:IDwpACtwqHLISdzfwUUNq4P9DsB/h5BLg4FwJPNfqFY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.26 h1:Si8kk1kyJnuJWCEgiwpBtTdtgSdR7i611596NnC0YIQ=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.26/go.mod h1:lBckz+W9SAdNtSDw3pYgQUJDJFcBBWry0GSzw+bK0TY=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 h1:/hi1JADLEW9YYryEz1w4GQu0EtP23pP553Cf9KgsDV4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30/go.mod h1:/3AOgy4K17Dm4ucMZVC/MJkzy5kmfKUcINRHZyo0koQ=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2/go.mod h1:wRQv0nN6v9wDXuWThpovGQjqF1HFdcgWjporw14lS8k=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 h1:f3vKqSo13fhTYb+JEcXwXefZQE26I1FB5eTSniU67ko=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29/go.mod h1:MzoLFUArKGpGD+ukmPiTPG1X5x4o6M2kq4v2dr1FiEc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 h1:xM/Is9cKMHa8Jj8zkvWhvrFkZsXJV9E+BB4g0HW0duQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30/go.mod h1:WueJeNDZvK1fMYEWJIkcivBfEzUkTpBhzlrUKKY8EuA=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2/go.mod h1:tyF5sKccmDz0Bv4NrstEr+/9YkSPJHrcO7UsUKf7pWM=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 h1:RdwIf/CuUsvJX3RgJagbOyotl/cxoLY4xviKuE7p2GY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29/go.mod h1:71wt8W2EgswdZy9Mf9KNnzxZ3TiZlv4caKghPktDOkA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 h1:jn46zC9LdsVR/ZpMIJqMqb8hHv31BlLx3ulVqNspUOk=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30/go.mod h1:1hTMsAgbdS/AtUi4bw8+gUuh1pceo+eXRLfpSuSQj3M=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 h1:VTGy885W5DKBxWRUJbym9hytNaYzsyaPkCHGRRMAOhU=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30/go.mod h1:AS0HycUvJRFvTt613AYDOgO2jzw+00cVSMny8XB3yMY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 h1:3GUprIsfmGcC5SACIyB0e7E0BM1O1b3Erl5CePYIAeQ=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31/go.mod h1:7PuV1yl5e2xnUbm+RqvVg5i2iBM8EyijZNoI9wsOoOc=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 h1:ZD2+BSw9vFsNlKYIasSNt3uDbjqqXIBcM13UJv/Lx2k=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12/go.mod h1:Ms4zlcVBbXbiP7EVLhl+lgjvA/a7YphqQ3Ih3174EmI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22 h1:V51LGlOq/1VsDsHUdoklAQi7rMmx4qQubvFYAlP2254=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22/go.mod h1:4Pzhyz8hJOm2bepgl+NjvRx8vlUFAIIvJnZ/MkcNPpU=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 h1:9Fjh6fi/U5JEStVZijmaMpUwE/gvBJj7x2B/PjbO9To=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23/go.mod h1:iMoT2f1tClxrWAAnKCXjZQ6LOmfLrMG14wmnWpM+F14=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 h1:DRebniUGZ2MqiiIVmQJ04vIXr918hubdHMnarSLEWyU=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29/go.mod h1:LfRkPCD8YHDM2E5eTkos2UpwYeZnBcVarTa8L59bJHA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 h1:hiME6pBzC7OTl9LMtlyTWBuEl1f4QBcUmFDKC7MLXtc=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29/go.mod h1:G7RP+uhagpKtKhd1BM9N6JQqjCcGEU47K5lBVZQyRQw=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0 h1:ta8csKy5vN91F3i5gGR85lFV0srBqySEji7Jroes6rE=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0/go.mod h1:77ZAgynvx1txMvDG8gGWoWkO1augYDxkp9JElWFgjQU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31/go.mod h1:I/1+z0VwL1GhQyLgkoHDlygpUZ+iTAwOQ/NsftiUL2I=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.104.2 h1:bAY6O/TDv1HQnvylh9E247IyIKsUWUt2G965S7qX110=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.104.2/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do=
+github.com/aws/aws-sdk-go-v2/service/signin v1.2.2 h1:69JEZSDTQ+UNbTWQJCZMmbpQb5sfc79KUt0O7Pyfjmo=
+github.com/aws/aws-sdk-go-v2/service/signin v1.2.2/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.3/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw=
 github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8=
 github.com/aws/aws-sdk-go-v2/service/sso v1.25.2/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.31.5 h1:xlK3Tdc8FO7Tq1k0+hL+otF33glj+dE+qeM5iINiDvU=
+github.com/aws/aws-sdk-go-v2/service/sso v1.31.5/go.mod h1:u8af9Nqkmqnr96f7v9nHqzZT9XBwbXEkTiqT4ROuJSE=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.3/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.8 h1:yX1IbiBfC7SdEgDwIGnRaZyPPDRbQPDOJxl8102PcGk=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.8/go.mod h1:DMPWJBjYs6+3+f/qhBFEFPPlQ6NlhWjai3dJNvipJ84=
 github.com/aws/aws-sdk-go-v2/service/sts v1.18.4/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s=
 github.com/aws/aws-sdk-go-v2/service/sts v1.28.1/go.mod h1:uQ7YYKZt3adCRrdCBREm1CD3efFLOUNH77MrUCvx5oA=
 github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
+github.com/aws/aws-sdk-go-v2/service/sts v1.43.5 h1:T3ANO8QWDbzQD8f4+UaX+fvJlyGnOFMKLbW+NGBHg04=
+github.com/aws/aws-sdk-go-v2/service/sts v1.43.5/go.mod h1:9gdl4RrflIdpDb2TlXshWgR1F9TeCkvqDx77Vpr4Z/Q=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
-github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8=
-github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY=
+github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/bazelbuild/rules_go v0.49.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs=
@@ -601,6 +613,8 @@ github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cenkalti/hub v1.0.2 h1:Nqv9TNaA9boeO2wQFW8o87BY3zKthtnzXmWGmJqhAV8=
 github.com/cenkalti/hub v1.0.2/go.mod h1:8LAFAZcCasb83vfxatMUnZHRoQcffho2ELpHb+kaTJU=
 github.com/cenkalti/rpc2 v1.0.5 h1:T6l4SS3ja3eaJfRyZrn7Oco/PSx/pr3YK5cjCgLVLTk=
@@ -1083,6 +1097,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9K
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
 github.com/hamba/avro/v2 v2.17.2/go.mod h1:Q9YK+qxAhtVrNqOhwlZTATLgLA8qxG2vtvkhK8fJ7Jo=
 github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
 github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
@@ -1185,8 +1201,8 @@ github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuOb
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
-github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
-github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
+github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
+github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
 github.com/klauspost/reedsolomon v1.14.1 h1:swE9kzyWXD/wVG+l5Pe8bWnQ0giIY7D1GjCBKk3kG2U=
 github.com/klauspost/reedsolomon v1.14.1/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -1287,10 +1303,10 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/predastore v1.10.0 h1:kMBI31HTYV0fxXYY+XiLgxO6rJd4WPPDXswDgJaK8HY=
-github.com/mulgadc/predastore v1.10.0/go.mod h1:j2DOdAOj/c42MRI6SkZrfh9KFbIyDDJLmf63SJ2yizw=
-github.com/mulgadc/viperblock v1.10.0 h1:a8KKJ/qgeYsInlnOJ5S/o+HCfqTIklgzz/FpzdeXAEY=
-github.com/mulgadc/viperblock v1.10.0/go.mod h1:2ArCxLXJjSN4Idx9g8Pdq5n1Kp7HFHs2vWMZgKJT9dI=
+github.com/mulgadc/predastore v1.11.0 h1:FD/Gk7cXhMZNcRBXectwBcR7jsMoS77WM3jcFgNVzY0=
+github.com/mulgadc/predastore v1.11.0/go.mod h1:gPrTLe9jY3CQyNtlpFurzUp2D0e8wPtVw/v48BmkhKA=
+github.com/mulgadc/viperblock v1.11.0 h1:/xIDWhwRIKkWiD1pvXH/f3aQe2SBH6MYQAlStNdek9Y=
+github.com/mulgadc/viperblock v1.11.0/go.mod h1:B4upzru+3opexPHNp6jkIqFvr8rdNs9lVVb+FJUd+9M=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -1300,8 +1316,8 @@ github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL
 github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
 github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
 github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k=
-github.com/nats-io/nats-server/v2 v2.14.2 h1:Q7dRhCY03Y00rETFW3KV+KGaCIajlDfWgWUVgbMxyuk=
-github.com/nats-io/nats-server/v2 v2.14.2/go.mod h1:lWpb1bSpRELZfRdlMkdz8E7lbXKKyNe8RIn0vvepIHs=
+github.com/nats-io/nats-server/v2 v2.14.3 h1:+xjydPt7rkit67G+04TN0mcO2n+8nveZE7tK/PPV53A=
+github.com/nats-io/nats-server/v2 v2.14.3/go.mod h1:5IlCtBzfwyzQzPMjmoJ9W2/LKmnJRtNyuOs/OT+NHDY=
 github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
 github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc=
 github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
@@ -1615,8 +1631,10 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.69.0 h1:MtkMsuRo3zEXTTMALfyrszwCDZTkB6wolyPjbwFAdq0=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.69.0/go.mod h1:FYTxnpsm+UPD0erZNq20GvnM8T2YQHiHtT2vokdpoac=
 go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
 go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
 go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
@@ -1640,6 +1658,12 @@ go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOX
 go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
 go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
 go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 h1:SUplec5dp06reu1zaXmOXdvqH398taqrDXqUl99jxSc=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0/go.mod h1:ho2g4N+ane+swq5I/VBkKWnRDY4kUINH3FuqyZqX/Ug=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo=
 go.opentelemetry.io/otel/exporters/prometheus v0.57.0/go.mod h1:QpFWz1QxqevfjwzYdbMb4Y1NnlJvqSGwyuU0B4iuc9c=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY=
@@ -1668,6 +1692,7 @@ go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9k
 go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
 go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
 go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
+go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA=
 go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk=
 go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
 go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
@@ -1737,6 +1762,8 @@ go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v8
 go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
 go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
 go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
+go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
+go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
@@ -2755,6 +2782,7 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.
 google.golang.org/genproto/googleapis/api v0.0.0-20260511170946-3700d4141b60/go.mod h1:7yoXV7RIh5gblj/xVYoogxAWvA9wUeVbpsK/M694l00=
 google.golang.org/genproto/googleapis/api v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:1dCETSCY2YKZNXQE3h4fun3TYwF5p8jejRKZgfWAgAY=
 google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
+google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3 h1:ctPmKL12ZsoKAlmPUsoW70zEDiYF+/H6aLieXxgAU0k=
 google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3/go.mod h1:Z4WJ5pJOYWFWcHEQUelD5QaZDknIQkpIL/+fyJOT9+A=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:+34luvCflYKiKylNwGJfn9cFBbcL/WrkciMmDmsTQ/A=

internal/eksgw/client.go

@@ -1,10 +1,11 @@
 // Package eksgw is the on-VM SigV4 client used by EKS helpers (eks-gateway-publish,
 // eks-token-webhook) to POST over HTTPS to the AWS gateway instead of NATS.
-// Signs with Predastore credentials for service "eks".
+// Signs for service "eks" with either static keys or IMDS instance-role creds.
 package eksgw
 
 import (
 	"bytes"
+	"context"
 	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
@@ -16,33 +17,50 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mulgadc/predastore/auth"
+	"github.com/mulgadc/spinifex/internal/gwsign"
 	"github.com/mulgadc/spinifex/internal/tlsconfig"
 )
 
 // Client posts SigV4-signed requests to the AWS gateway for the "eks" service.
 type Client struct {
 	baseURL    string
-	accessKey  string
-	secretKey  string
+	signer     *gwsign.Signer
 	region     string
 	httpClient *http.Client
 }
 
 // New builds a client. caPath optionally pins the gateway TLS CA; when empty the
 // client relies on the system trust store. region defaults to us-east-1 when
-// empty (SigV4 requires a non-empty region).
+// empty (SigV4 requires a non-empty region). When accessKey/secretKey are empty
+// the client signs with IMDS instance-role credentials served in-VM.
 func New(baseURL, caPath, accessKey, secretKey, region string) (*Client, error) {
 	if baseURL == "" {
 		return nil, fmt.Errorf("eksgw: baseURL is required")
 	}
-	if accessKey == "" || secretKey == "" {
-		return nil, fmt.Errorf("eksgw: access key and secret key are required")
-	}
 	if region == "" {
 		region = "us-east-1"
 	}
 
+	var signer *gwsign.Signer
+	if accessKey != "" && secretKey != "" {
+		signer = gwsign.NewStatic(accessKey, secretKey)
+	} else {
+		s, err := gwsign.NewIMDS(context.Background(), region)
+		if err != nil {
+			return nil, fmt.Errorf("eksgw: init IMDS signer: %w", err)
+		}
+		// The per-tap IMDS datapath lags VM boot by minutes (data-NIC DHCP lease
+		// + br-imds reconcile), and the SDK's IMDS client fast-fails each probe.
+		// Block until the instance-role creds first retrieve so the first signed
+		// request already has them, rather than letting each caller burn its own
+		// short POST-retry budget against a still-cold datapath (observed: the CP
+		// datapath warms at ~264s, past gateway-publish's 150s budget).
+		if err := warmIMDSCredentials(s); err != nil {
+			return nil, fmt.Errorf("eksgw: IMDS instance-role credentials unavailable: %w", err)
+		}
+		signer = s
+	}
+
 	tlsCfg := &tls.Config{
 		MinVersion:       tls.VersionTLS13,
 		CurvePreferences: tlsconfig.Curves,
@@ -60,10 +78,9 @@ func New(baseURL, caPath, accessKey, secretKey, region string) (*Client, error)
 	}
 
 	return &Client{
-		baseURL:   strings.TrimRight(baseURL, "/"),
-		accessKey: accessKey,
-		secretKey: secretKey,
-		region:    region,
+		baseURL: strings.TrimRight(baseURL, "/"),
+		signer:  signer,
+		region:  region,
 		httpClient: &http.Client{
 			Timeout:   10 * time.Second,
 			Transport: &http.Transport{TLSClientConfig: tlsCfg, MaxIdleConns: 2, IdleConnTimeout: 30 * time.Second},
@@ -71,6 +88,37 @@ func New(baseURL, caPath, accessKey, secretKey, region string) (*Client, error)
 	}, nil
 }
 
+const (
+	// imdsWarmupTimeout bounds the wait for the IMDS instance-role datapath to
+	// come up — aligned with cloud-init's Ec2 max_wait (600s), since it is the
+	// same per-tap datapath. Exceeding it means the datapath is genuinely broken
+	// (not merely slow), so New fails loudly instead of signing credential-less.
+	imdsWarmupTimeout = 10 * time.Minute
+	// imdsWarmupInterval spaces the retrieve probes; each SDK probe fast-fails
+	// (~250ms dial) against a cold datapath, so a short interval keeps the total
+	// wait close to the datapath's actual readiness time.
+	imdsWarmupInterval = 2 * time.Second
+)
+
+// warmIMDSCredentials blocks until the signer's IMDS provider first yields
+// credentials, or imdsWarmupTimeout elapses. Retries because the SDK client
+// fast-fails each probe while the per-tap datapath is still cold.
+func warmIMDSCredentials(s *gwsign.Signer) error {
+	ctx, cancel := context.WithTimeout(context.Background(), imdsWarmupTimeout)
+	defer cancel()
+	var lastErr error
+	for {
+		if lastErr = s.EnsureCredentials(ctx); lastErr == nil {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("after %s: %w", imdsWarmupTimeout, lastErr)
+		case <-time.After(imdsWarmupInterval):
+		}
+	}
+}
+
 // Post SigV4-signs and sends body to path, returning the response body on 2xx.
 // Errors include the gateway status and body. No retry; callers wrap as needed.
 func (c *Client) Post(path string, body []byte) ([]byte, error) {
@@ -94,7 +142,7 @@ func (c *Client) do(method, path string, body []byte) ([]byte, error) {
 	}
 
 	sum := sha256.Sum256(body)
-	if err := auth.SignReq(req, c.accessKey, c.secretKey, hex.EncodeToString(sum[:]), "eks", c.region); err != nil {
+	if err := c.signer.Sign(req, hex.EncodeToString(sum[:]), "eks", c.region); err != nil {
 		return nil, fmt.Errorf("sign request: %w", err)
 	}
 

internal/gwsign/signer.go

@@ -0,0 +1,70 @@
+// Package gwsign SigV4-signs on-VM requests to the AWS gateway. Credentials
+// come from either fixed long-lived keys or the AWS SDK default credential
+// chain, which resolves the in-VM IMDS instance-role credentials and rotates
+// them automatically. A session token, when present, is signed as
+// X-Amz-Security-Token so the gateway can verify temporary (ASIA) credentials.
+package gwsign
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+)
+
+// Signer holds a credential provider and signs requests in place.
+type Signer struct {
+	provider aws.CredentialsProvider
+}
+
+// NewStatic returns a Signer using fixed long-lived keys (no session token).
+func NewStatic(accessKey, secretKey string) *Signer {
+	return &Signer{provider: credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")}
+}
+
+// NewIMDS returns a Signer backed by the AWS SDK default credential chain. In a
+// guest VM with no env/profile credentials the chain falls back to the IMDS
+// endpoint (169.254.169.254), yielding scoped, auto-rotating instance-role
+// credentials. region sets the chain's default region.
+func NewIMDS(ctx context.Context, region string) (*Signer, error) {
+	cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region))
+	if err != nil {
+		return nil, fmt.Errorf("load AWS default config: %w", err)
+	}
+	if cfg.Credentials == nil {
+		return nil, fmt.Errorf("default credential chain resolved no provider")
+	}
+	return &Signer{provider: cfg.Credentials}, nil
+}
+
+// EnsureCredentials retrieves once from the provider, surfacing the error so a
+// caller can warm the IMDS instance-role datapath before the first Sign. The
+// SDK's IMDS client fast-fails each probe (250ms dial), so a cold per-tap
+// datapath returns "context deadline exceeded" / "no EC2 IMDS role found" here;
+// a caller loops until this succeeds. A static signer always succeeds.
+func (s *Signer) EnsureCredentials(ctx context.Context) error {
+	if _, err := s.provider.Retrieve(ctx); err != nil {
+		return fmt.Errorf("retrieve credentials: %w", err)
+	}
+	return nil
+}
+
+// Sign signs r in place for service/region. payloadHash is the hex SHA-256 of
+// the body (or a SigV4 sentinel). Credentials are retrieved per call so rotated
+// IMDS credentials take effect without a restart; the SDK signer sets
+// X-Amz-Security-Token when the credentials carry a session token.
+func (s *Signer) Sign(r *http.Request, payloadHash, service, region string) error {
+	creds, err := s.provider.Retrieve(r.Context())
+	if err != nil {
+		return fmt.Errorf("retrieve credentials: %w", err)
+	}
+	// The gateway recovers the payload hash from this header for non-S3
+	// services; the SDK signer alone does not set it for direct callers.
+	r.Header.Set("X-Amz-Content-Sha256", payloadHash)
+	return v4.NewSigner().SignHTTP(r.Context(), creds, r, payloadHash, service, region, time.Now().UTC())
+}

internal/gwsign/signer_test.go

@@ -0,0 +1,124 @@
+package gwsign
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/mulgadc/predastore/auth"
+)
+
+// failingProvider models a cold IMDS datapath: Retrieve always errors, as the
+// SDK's IMDS client does before the per-tap datapath is up.
+type failingProvider struct{ err error }
+
+func (p failingProvider) Retrieve(context.Context) (aws.Credentials, error) {
+	return aws.Credentials{}, p.err
+}
+
+const (
+	testAccessKey = "AKIAIOSFODNN7EXAMPLE"
+	testSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+	testRegion    = "us-east-1"
+	testService   = "elasticloadbalancing"
+)
+
+// TestSign_StaticVerifies confirms a statically-keyed Signer produces a request
+// the gateway's SigV4 verifier accepts, with X-Amz-Content-Sha256 set to the
+// body hash.
+func TestSign_StaticVerifies(t *testing.T) {
+	body := []byte("Action=LBAgentHeartbeat")
+	sum := sha256.Sum256(body)
+	payloadHash := hex.EncodeToString(sum[:])
+
+	var verifyErr error
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		req, err := auth.ParseReq(r)
+		if err != nil {
+			verifyErr = err
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		verifyErr = req.Verify(testSecretKey, testService, testRegion, auth.WithBodyHash(payloadHash))
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	req, _ := http.NewRequest(http.MethodPost, srv.URL, strings.NewReader(string(body)))
+	if err := NewStatic(testAccessKey, testSecretKey).Sign(req, payloadHash, testService, testRegion); err != nil {
+		t.Fatalf("Sign: %v", err)
+	}
+	if got := req.Header.Get("X-Amz-Content-Sha256"); got != payloadHash {
+		t.Fatalf("X-Amz-Content-Sha256 = %q, want %q", got, payloadHash)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("do: %v", err)
+	}
+	resp.Body.Close()
+	if verifyErr != nil {
+		t.Fatalf("server-side verify failed: %v", verifyErr)
+	}
+}
+
+// TestSign_SessionTokenHeader confirms temporary credentials surface as a signed
+// X-Amz-Security-Token header — the path the gateway uses to verify ASIA
+// instance-role credentials.
+func TestSign_SessionTokenHeader(t *testing.T) {
+	const token = "FwoGZXIvYXdzEXAMPLEsessiontoken"
+	s := &Signer{provider: credentials.NewStaticCredentialsProvider(testAccessKey, testSecretKey, token)}
+
+	body := []byte("Action=LBAgentHeartbeat")
+	sum := sha256.Sum256(body)
+	req, _ := http.NewRequest(http.MethodPost, "https://gw:9999/", strings.NewReader(string(body)))
+	if err := s.Sign(req, hex.EncodeToString(sum[:]), testService, testRegion); err != nil {
+		t.Fatalf("Sign: %v", err)
+	}
+
+	if got := req.Header.Get("X-Amz-Security-Token"); got != token {
+		t.Fatalf("X-Amz-Security-Token = %q, want %q", got, token)
+	}
+	authHdr := req.Header.Get("Authorization")
+	if !strings.Contains(authHdr, "x-amz-security-token") {
+		t.Fatalf("session token not in SignedHeaders: %q", authHdr)
+	}
+}
+
+// TestEnsureCredentials confirms a resolvable provider warms clean while a cold
+// datapath surfaces its retrieve error (the signal a warm-up loop retries on).
+func TestEnsureCredentials(t *testing.T) {
+	ok := &Signer{provider: credentials.NewStaticCredentialsProvider(testAccessKey, testSecretKey, "")}
+	if err := ok.EnsureCredentials(t.Context()); err != nil {
+		t.Fatalf("EnsureCredentials(static) = %v, want nil", err)
+	}
+
+	coldErr := errors.New("no EC2 IMDS role found")
+	cold := &Signer{provider: failingProvider{err: coldErr}}
+	err := cold.EnsureCredentials(t.Context())
+	if err == nil {
+		t.Fatal("EnsureCredentials(cold) = nil, want error")
+	}
+	if !errors.Is(err, coldErr) {
+		t.Fatalf("EnsureCredentials(cold) = %v, want wrap of %v", err, coldErr)
+	}
+}
+
+// TestNewIMDS_BuildsProvider confirms the IMDS constructor resolves a provider
+// without performing network I/O (the chain retrieves lazily at first sign).
+func TestNewIMDS_BuildsProvider(t *testing.T) {
+	s, err := NewIMDS(t.Context(), testRegion)
+	if err != nil {
+		t.Fatalf("NewIMDS: %v", err)
+	}
+	if s.provider == nil {
+		t.Fatal("expected non-nil credentials provider")
+	}
+}

scripts/images/eks-node/cloud-init-ec2.cfg

@@ -0,0 +1,25 @@
+# Spinifex eks-node cloud-init datasource pin (installed to
+# /etc/cloud/cloud.cfg.d/90-spinifex-ec2-imds.cfg).
+#
+# The system VM's user-data is delivered only via IMDS (the NoCloud seed
+# was retired in #411), and IMDS (169.254.169.254) is reachable only once
+# mulga-mgmt-net has brought up the data NIC via DHCP and the per-tap
+# br-imds datapath has reconciled. On the first-launched (primary) control
+# plane that datapath can lag cloud-init's stock 120s probe window, so with
+# the default max_wait cloud-init gives up and falls back to DataSourceNone
+# — permanently losing the launch user-data (no SPINIFEX_K3S_ROLE → no k3s
+# server). Raise max_wait so cloud-init keeps polling IMDS until the
+# datapath is ready instead of falling through to None.
+datasource_list: [ Ec2, None ]
+datasource:
+  Ec2:
+    # Poll IMDS up to 10 min (default 120s). Well past the ~264s at which
+    # the datapath became reachable in the observed race; a slow boot, never
+    # a lost role.
+    max_wait: 600
+    timeout: 20
+    # DMI is spoofed to an ec2 id by the launcher; don't abort if a strict
+    # id check ever misfires.
+    strict_id: false
+    metadata_urls:
+      - http://169.254.169.254

scripts/images/eks-node/coredns/coredns-mulga.yaml

@@ -1,10 +1,11 @@
 # Mulga-owned CoreDNS for the eks-node AMI. Replaces k3s's bundled CoreDNS
 # (suppressed via a coredns.yaml.skip marker staged by k3s.initd) so DNS is
 # node-local and never depends on the cross-node pod overlay: a DaemonSet runs
-# CoreDNS on every node and kube-dns uses internalTrafficPolicy=Local, so each
-# node resolves through its own CoreDNS. In the managed-control-plane topology
-# the worker<->CP overlay cannot cross the VPC boundary, so a CP-resident
-# CoreDNS is unreachable from worker pods.
+# CoreDNS on every worker node and kube-dns uses internalTrafficPolicy=Local, so
+# each worker resolves through its own CoreDNS. Mirroring AWS EKS, CoreDNS runs
+# on workers only (nodeAffinity excludes control-plane): the managed control
+# plane never hosts kube-dns, and the worker<->CP overlay cannot cross the VPC
+# boundary, so a CP-resident CoreDNS would be unreachable from worker pods.
 #
 # Faithful to k3s v1.32.5+k3s1 (image, Corefile, RBAC, Service) except the two
 # locality changes above. Refresh image/Corefile when bumping K3S_VERSION in
@@ -118,19 +119,20 @@ spec:
     spec:
       priorityClassName: system-cluster-critical
       serviceAccountName: coredns
-      # CriticalAddonsOnly (Exists, all effects) tolerates the CP's
-      # CriticalAddonsOnly=true:NoExecute taint so a CoreDNS pod also runs on
-      # the control-plane node; the control-plane/master tolerations cover any
-      # role taints. Required for full per-node coverage with internalTrafficPolicy=Local.
-      tolerations:
-        - key: CriticalAddonsOnly
-          operator: Exists
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-        - key: node-role.kubernetes.io/master
-          operator: Exists
-          effect: NoSchedule
+      # AWS-EKS parity: CoreDNS runs on worker nodes only. The control plane is
+      # managed and never hosts kube-dns; each worker resolves through its own
+      # node's CoreDNS (internalTrafficPolicy=Local), so DNS never crosses the
+      # worker<->CP overlay boundary. No CP toleration is carried, and
+      # anti-control-plane nodeAffinity keeps the DaemonSet off the K3s server VMs.
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: DoesNotExist
+                  - key: node-role.kubernetes.io/master
+                    operator: DoesNotExist
       nodeSelector:
         kubernetes.io/os: linux
       containers:

scripts/images/eks-node/eks-node-role.sh

@@ -52,7 +52,15 @@ case "${ROLE}" in
     server)
         log "configuring server role"
         rc-update add eks-token-webhook default
+        # Pre-k3s etcd recovery: added to the runlevel so it runs before k3s on
+        # every subsequent boot, but not started now — a fresh cluster has no
+        # directive and the role file it guards on is written at the end of this
+        # script. It first acts on the reboot the reconciler triggers.
+        rc-update add mulga-eks-k3s-recovery default
         rc-update add k3s default
+        # konnectivity-server fronts apiserver egress; every server replica runs
+        # one so the agent's per-replica tunnel always lands a live server.
+        rc-update add konnectivity-server default
         rc-update add k3s-first-boot default
         rc-update add mulga-eks-state-report default
         # Managed-addon delivery runs only on the primary server: it renders
@@ -62,6 +70,7 @@ case "${ROLE}" in
         rc-update add mulga-eks-addon-sync default
         rc-service eks-token-webhook start
         rc-service k3s start
+        rc-service konnectivity-server start
         rc-service k3s-first-boot start
         rc-service mulga-eks-state-report start
         rc-service mulga-eks-addon-sync start
@@ -69,10 +78,15 @@ case "${ROLE}" in
     server-join)
         log "configuring server-join role"
         rc-update add eks-token-webhook default
+        # Pre-k3s etcd recovery (see server role): server-join members are the
+        # wipe-rejoin followers of a cluster-reset, so they need it too.
+        rc-update add mulga-eks-k3s-recovery default
         rc-update add k3s default
+        rc-update add konnectivity-server default
         rc-update add mulga-eks-state-report default
         rc-service eks-token-webhook start
         rc-service k3s start
+        rc-service konnectivity-server start
         rc-service mulga-eks-state-report start
         ;;
     agent)

scripts/images/eks-node/eks-node-role_test.sh

@@ -63,6 +63,10 @@ check "server: enables webhook" "yes" "$(grep -q 'rc-update add eks-token-webhoo
 check "server: enables k3s" "yes" "$(grep -q 'rc-update add k3s default' "${CALLS}" && echo yes || echo no)"
 check "server: enables k3s-first-boot" "yes" "$(grep -q 'rc-update add k3s-first-boot default' "${CALLS}" && echo yes || echo no)"
 check "server: starts k3s" "yes" "$(grep -q 'rc-service k3s start' "${CALLS}" && echo yes || echo no)"
+check "server: enables konnectivity-server" "yes" "$(grep -q 'rc-update add konnectivity-server default' "${CALLS}" && echo yes || echo no)"
+check "server: enables k3s-recovery" "yes" "$(grep -q 'rc-update add mulga-eks-k3s-recovery default' "${CALLS}" && echo yes || echo no)"
+# Recovery is added to the runlevel (pre-k3s on later boots) but not started now.
+check "server: does not start k3s-recovery" "no" "$(grep -q 'rc-service mulga-eks-k3s-recovery' "${CALLS}" && echo yes || echo no)"
 check "server: no k3s-agent" "no" "$(grep -q 'k3s-agent' "${CALLS}" && echo yes || echo no)"
 check "server: self-disables" "yes" "$(grep -q 'rc-update del eks-node-role default' "${CALLS}" && echo yes || echo no)"
 
@@ -75,6 +79,8 @@ check "server-join: role file" "server-join" "$(cat "${ROLE_FILE}")"
 check "server-join: enables webhook" "yes" "$(grep -q 'rc-update add eks-token-webhook default' "${CALLS}" && echo yes || echo no)"
 check "server-join: enables k3s" "yes" "$(grep -q 'rc-update add k3s default' "${CALLS}" && echo yes || echo no)"
 check "server-join: enables state-report" "yes" "$(grep -q 'rc-update add mulga-eks-state-report default' "${CALLS}" && echo yes || echo no)"
+check "server-join: enables konnectivity-server" "yes" "$(grep -q 'rc-update add konnectivity-server default' "${CALLS}" && echo yes || echo no)"
+check "server-join: enables k3s-recovery" "yes" "$(grep -q 'rc-update add mulga-eks-k3s-recovery default' "${CALLS}" && echo yes || echo no)"
 # Join servers must NOT re-publish bootstrap; the first server already did.
 check "server-join: no k3s-first-boot" "no" "$(grep -q 'k3s-first-boot' "${CALLS}" && echo yes || echo no)"
 check "server-join: no k3s-agent" "no" "$(grep -q 'k3s-agent' "${CALLS}" && echo yes || echo no)"

scripts/images/eks-node/k3s-first-boot.sh

@@ -48,7 +48,9 @@ set -a
 . "${ENVFILE}"
 set +a
 
-for v in EKS_GATEWAY_URL EKS_ACCESS_KEY EKS_SECRET_KEY EKS_ACCOUNT_ID EKS_CLUSTER_NAME EKS_NLB_ENDPOINT; do
+# EKS_ACCESS_KEY/EKS_SECRET_KEY are intentionally not required: when absent,
+# eks-gateway-publish signs with IMDS instance-role creds via the SDK chain.
+for v in EKS_GATEWAY_URL EKS_ACCOUNT_ID EKS_CLUSTER_NAME EKS_NLB_ENDPOINT; do
     eval "val=\${$v:-}"
     [ -n "${val}" ] || die "env ${v} not set"
 done
@@ -133,6 +135,29 @@ jq -n --arg k "${KUBECONFIG_REWRITTEN}" '{adminKubeconfig: $k}'      | publish_e
 jq -n --arg j "${JWKS}"                 '{jwks: $j}'                  | publish_envelope k3s-oidc-jwks
 jq -n --arg c "${CA_B64}"               '{certificateAuthority: $c}' | publish_envelope k3s-ca
 
+# 3.5 Prune a terminated control-plane peer this VM replaces. The spinifex
+# reconciler sets EKS_ETCD_PRUNE_PEER_IP to the dead member's node IP when this
+# VM is provisioned as its replacement (member-count reconcile). Deleting the
+# dead Node makes k3s embedded-etcd remove the stale member, so quorum width
+# returns to N rather than N+1-with-a-dead-peer. Best-effort: a failure leaves
+# the member for an operator and never blocks first boot.
+if [ -n "${EKS_ETCD_PRUNE_PEER_IP:-}" ]; then
+    log "pruning terminated CP peer at ${EKS_ETCD_PRUNE_PEER_IP} from etcd"
+    dead_node=$(kubectl --kubeconfig "${KUBECONFIG_FILE}" get nodes \
+        -o "custom-columns=NAME:.metadata.name,IP:.status.addresses[?(@.type=='InternalIP')].address" \
+        --no-headers 2>/dev/null \
+        | awk -v ip="${EKS_ETCD_PRUNE_PEER_IP}" '$2==ip {print $1; exit}')
+    if [ -n "${dead_node}" ]; then
+        if kubectl --kubeconfig "${KUBECONFIG_FILE}" delete node "${dead_node}" --ignore-not-found=true; then
+            log "deleted dead node ${dead_node}; k3s will remove its etcd member"
+        else
+            log "WARN: delete of dead node ${dead_node} failed; etcd member left for operator"
+        fi
+    else
+        log "no node has internal IP ${EKS_ETCD_PRUNE_PEER_IP}; nothing to prune"
+    fi
+fi
+
 # 4. Self-disable. Remove sentinel, pull from runlevel.
 rm -f "${SENTINEL}"
 rc-update del k3s-first-boot default 2>/dev/null || true

scripts/images/eks-node/k3s.initd

@@ -8,6 +8,11 @@ command="/usr/local/bin/k3s"
 command_args="server --config /etc/rancher/k3s/config.yaml"
 pidfile="/run/k3s.pid"
 
+# Konnectivity agent image (apiserver-network-proxy). Pinned alongside the
+# konnectivity-server version baked by setup.sh; staged into the worker
+# DaemonSet by start_pre.
+KONN_AGENT_IMAGE="registry.k8s.io/kas-network-proxy/proxy-agent:v0.30.3"
+
 # Supervise the apiserver. A transient k3s crash must self-heal: without a
 # supervisor a single exit leaves the control plane permanently dead, the NLB
 # target never recovers, and the cluster wedges ACTIVE-but-unreachable.
@@ -34,6 +39,28 @@ start_pre() {
         fi
     fi
 
+    # Konnectivity agent: stage the worker-side DaemonSet into the auto-deploy dir
+    # (the apiserver `cluster` egress selector tunnels through konnectivity-server,
+    # which the agents dial out to). __KONN_HOST__ is the NLB endpoint agents reach,
+    # seeded by the launcher in first-boot.env; __KONN_AGENT_IMAGE__ is pinned here.
+    # The agent learns the apiserver replica count from the server handshake
+    # (--sync-forever), so no server-count token is substituted here. Guarded on
+    # the baked manifest so an older image without it simply skips the agent.
+    if [ -f /usr/share/spinifex-eks/konnectivity/konnectivity-agent.yaml ]; then
+        konn_host=$(awk -F= '/^EKS_KONNECTIVITY_HOST=/{print $2}' \
+            /etc/spinifex-eks/first-boot.env 2>/dev/null)
+        if [ -n "${konn_host}" ]; then
+            mkdir -p /var/lib/rancher/k3s/server/manifests
+            sed -e "s|__KONN_HOST__|${konn_host}|g" \
+                -e "s|__KONN_AGENT_IMAGE__|${KONN_AGENT_IMAGE}|g" \
+                /usr/share/spinifex-eks/konnectivity/konnectivity-agent.yaml \
+                > /var/lib/rancher/k3s/server/manifests/konnectivity-agent.yaml
+            einfo "konnectivity-agent staged (host ${konn_host})"
+        else
+            ewarn "EKS_KONNECTIVITY_HOST empty; konnectivity-agent not staged, apiserver->pod egress will fail"
+        fi
+    fi
+
     # Node-local CoreDNS: suppress k3s's bundled CoreDNS (a single Deployment that
     # tolerates the CP taint and strands itself on the control-plane node) and
     # deploy the Mulga DaemonSet variant so every node resolves via its own

scripts/images/eks-node/konnectivity-server.initd

@@ -0,0 +1,94 @@
+#!/sbin/openrc-run
+# Konnectivity server (apiserver-network-proxy) — the apiserver egress proxy for
+# the Mulga EKS control plane. The kube-apiserver's `cluster` egress selector
+# (egress-selector-config.yaml) routes pod/kubelet/webhook traffic to this
+# server's UDS; konnectivity-agents on the workers dial in over :8132 and the
+# egress rides back down the agent-initiated reverse tunnel. Server-role only;
+# enabled by eks-node-role. Runs as root to bind the UDS the apiserver reads.
+
+description="Konnectivity server (Mulga EKS apiserver egress proxy)"
+
+KONN_DIR=/var/lib/spinifex-eks/konnectivity
+KONN_UDS=/run/konnectivity/konnectivity-server.socket
+ENVFILE=/etc/spinifex-eks/first-boot.env
+K3S_CA=/var/lib/rancher/k3s/server/tls/server-ca.crt
+K3S_CA_KEY=/var/lib/rancher/k3s/server/tls/server-ca.key
+KUBECONFIG_FILE=/etc/rancher/k3s/k3s.yaml
+
+# server-count is the apiserver replica count: each agent holds a tunnel to every
+# replica, so egress from any apiserver always resolves (the HA-correct fan-out).
+# Seeded by the launcher in first-boot.env; default 1 for a single control plane.
+_server_count=1
+if [ -f "${ENVFILE}" ]; then
+    _sc=$(awk -F= '/^EKS_KONNECTIVITY_SERVER_COUNT=/{print $2}' "${ENVFILE}")
+    [ -n "${_sc}" ] && _server_count="${_sc}"
+fi
+
+command="/usr/local/bin/konnectivity-server"
+command_args="--logtostderr=true
+ --uds-name=${KONN_UDS}
+ --delete-existing-uds-file
+ --cluster-cert=${KONN_DIR}/tls.crt
+ --cluster-key=${KONN_DIR}/tls.key
+ --server-count=${_server_count}
+ --mode=grpc
+ --server-port=0
+ --agent-port=8132
+ --admin-port=8093
+ --health-port=8092
+ --kubeconfig=${KUBECONFIG_FILE}
+ --authentication-audience=system:konnectivity-server
+ --agent-namespace=kube-system
+ --agent-service-account=konnectivity-agent
+ --proxy-strategies=destHost,default"
+pidfile="/run/konnectivity-server.pid"
+
+# Supervise: a transient exit must self-heal, or apiserver egress (webhooks,
+# exec, logs) stays broken until the next boot and the cluster wedges.
+supervisor="supervise-daemon"
+respawn_delay=5
+respawn_max=0
+output_log="/var/log/konnectivity-server.log"
+error_log="/var/log/konnectivity-server.log"
+
+depend() {
+    need net
+    after k3s
+}
+
+start_pre() {
+    mkdir -p /run/konnectivity "${KONN_DIR}"
+
+    # Wait for k3s to write the server CA (+ key) and admin kubeconfig: the CA
+    # signs the serving cert agents validate, the kubeconfig is the TokenReview
+    # client for agent SA-token auth. Bounded; without them konnectivity is inert.
+    i=0
+    while [ "${i}" -lt 120 ]; do
+        if [ -r "${K3S_CA}" ] && [ -r "${K3S_CA_KEY}" ] && [ -r "${KUBECONFIG_FILE}" ]; then
+            break
+        fi
+        i=$((i + 2))
+        sleep 2
+    done
+    if [ ! -r "${K3S_CA}" ] || [ ! -r "${K3S_CA_KEY}" ] || [ ! -r "${KUBECONFIG_FILE}" ]; then
+        eerror "k3s server CA / admin kubeconfig not present after ${i}s"
+        return 1
+    fi
+
+    # Mint (or reuse) the serving cert from the k3s server CA, SANed to the NLB
+    # endpoints agents dial. That CA is the in-pod ca.crt the agent already trusts,
+    # so no separate CA distribution is needed.
+    sans=""
+    if [ -f "${ENVFILE}" ]; then
+        sans=$(awk -F= '/^EKS_KONNECTIVITY_SANS=/{print $2}' "${ENVFILE}")
+    fi
+    if [ -z "${sans}" ]; then
+        eerror "EKS_KONNECTIVITY_SANS empty; agents would fail server cert validation"
+        return 1
+    fi
+    if ! eks-konnectivity-cert -dir "${KONN_DIR}" -ca-cert "${K3S_CA}" \
+            -ca-key "${K3S_CA_KEY}" -cn konnectivity-server -sans "${sans}" >/dev/null; then
+        eerror "failed to mint konnectivity serving cert"
+        return 1
+    fi
+}

scripts/images/eks-node/konnectivity/konnectivity-agent.yaml

@@ -0,0 +1,89 @@
+# Konnectivity agent DaemonSet — the worker half of the apiserver egress proxy.
+# Each agent dials OUT to the konnectivity-server (fronted by the cluster NLB at
+# __KONN_HOST__:8132) and, via --sync-forever, opens a reverse tunnel to EVERY
+# apiserver replica (it learns the replica count from the server handshake).
+# apiserver→pod/kubelet/webhook egress rides back down that tunnel, so an HA
+# control plane reaches worker pod IPs with no L4-VIP fan-out gap. Staged into
+# the K3s auto-deploy dir by k3s.initd, which fills the __KONN_HOST__ /
+# __KONN_AGENT_IMAGE__ tokens at boot.
+#
+# Runs on workers only: it carries no toleration for the control-plane
+# CriticalAddonsOnly=true:NoExecute taint, so it is never scheduled onto the CP
+# nodes (which run the server, not an agent).
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: konnectivity-agent
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/managed-by: spinifex-eks
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: konnectivity-agent
+  namespace: kube-system
+  labels:
+    k8s-app: konnectivity-agent
+    app.kubernetes.io/managed-by: spinifex-eks
+spec:
+  selector:
+    matchLabels:
+      k8s-app: konnectivity-agent
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: konnectivity-agent
+    spec:
+      serviceAccountName: konnectivity-agent
+      priorityClassName: system-node-critical
+      # hostNetwork so the agent reaches the NLB over the node network and
+      # identifies itself to the server by the node's own IP (the address the
+      # apiserver dials back through for kubelet/pod egress).
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+        - name: konnectivity-agent
+          image: __KONN_AGENT_IMAGE__
+          command:
+            - /proxy-agent
+          args:
+            - --logtostderr=true
+            - --ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+            - --proxy-server-host=__KONN_HOST__
+            - --proxy-server-port=8132
+            # proxy-agent v0.30.3 has no server-count flag — it learns the
+            # replica count from the server handshake and keeps re-syncing
+            # (sync-forever) until it holds a tunnel to every apiserver, which
+            # is what makes the HA control plane reachable.
+            - --sync-forever=true
+            - --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token
+            - --agent-identifiers=ipv4=$(HOST_IP)
+            - --admin-server-port=8133
+            - --health-server-port=8134
+          env:
+            - name: HOST_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+          livenessProbe:
+            httpGet:
+              scheme: HTTP
+              host: 127.0.0.1
+              port: 8134
+              path: /healthz
+            initialDelaySeconds: 15
+            timeoutSeconds: 15
+          volumeMounts:
+            - name: konnectivity-agent-token
+              mountPath: /var/run/secrets/tokens
+              readOnly: true
+      volumes:
+        - name: konnectivity-agent-token
+          projected:
+            sources:
+              - serviceAccountToken:
+                  path: konnectivity-agent-token
+                  audience: system:konnectivity-server

scripts/images/eks-node/manifest.conf

@@ -1,5 +1,5 @@
 IMAGE_NAME="eks-node"
-IMAGE_DESCRIPTION="Unified EKS K3s node VM — Alpine + K3s v1.32.5+k3s1 + eks-token-webhook; role (server|agent) chosen at first boot"
+IMAGE_DESCRIPTION="Unified EKS K3s node VM — Alpine + K3s v1.32.5+k3s1 + eks-token-webhook + konnectivity-server v0.30.3; role (server|agent) chosen at first boot"
 DISTRO="alpine"
 ALPINE_VERSION="3.21.7"
 
@@ -18,9 +18,13 @@ APK_PACKAGES="ca-certificates curl jq iptables ip6tables conntrack-tools cgroup-
 # eks-gateway-publish relays the bootstrap envelopes + state reports to the AWS
 # gateway over SigV4 HTTPS (server role only). The k3s binary is downloaded +
 # SHA256-verified by setup.sh (network in chroot).
-BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-fetch ./cmd/eks-gateway-fetch && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-webhook-cert ./cmd/eks-webhook-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/ecr-credential-provider ./cmd/ecr-credential-provider"
+# konnectivity-server is built from the upstream apiserver-network-proxy source
+# at a pinned tag (its GitHub releases ship only container images, and its go.mod
+# carries replace directives that block `go install pkg@version`). The agent half
+# is the matching proxy-agent container image, pinned in k3s.initd.
+BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-fetch ./cmd/eks-gateway-fetch && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-webhook-cert ./cmd/eks-webhook-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-konnectivity-cert ./cmd/eks-konnectivity-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/ecr-credential-provider ./cmd/ecr-credential-provider && rm -rf /tmp/anp-build && git clone --depth 1 --branch v0.30.3 https://github.com/kubernetes-sigs/apiserver-network-proxy /tmp/anp-build && ( cd /tmp/anp-build && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o /tmp/anp-build/konnectivity-server ./cmd/server ) && cp /tmp/anp-build/konnectivity-server bin/konnectivity-server && rm -rf /tmp/anp-build"
 
-INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish bin/eks-gateway-fetch:/usr/local/bin/eks-gateway-fetch bin/eks-webhook-cert:/usr/local/bin/eks-webhook-cert bin/ecr-credential-provider:/usr/local/bin/ecr-credential-provider"
+INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish bin/eks-gateway-fetch:/usr/local/bin/eks-gateway-fetch bin/eks-webhook-cert:/usr/local/bin/eks-webhook-cert bin/eks-konnectivity-cert:/usr/local/bin/eks-konnectivity-cert bin/konnectivity-server:/usr/local/bin/konnectivity-server bin/ecr-credential-provider:/usr/local/bin/ecr-credential-provider"
 
 # All role services ship; none but the selector are enabled at bake time.
 # eks-node-role picks server vs agent at first boot and enables the rest.
@@ -36,12 +40,18 @@ scripts/images/eks-node/mulga-eks-state-report.initd:/etc/init.d/mulga-eks-state
 scripts/images/eks-node/mulga-eks-state-report.sh:/usr/local/sbin/mulga-eks-state-report \
 scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-sync \
 scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \
+scripts/images/eks-node/konnectivity-server.initd:/etc/init.d/konnectivity-server \
+scripts/images/eks-node/konnectivity/konnectivity-agent.yaml:/usr/share/spinifex-eks/konnectivity/konnectivity-agent.yaml \
 scripts/images/eks-node/mulga-ebs-byid.initd:/etc/init.d/mulga-ebs-byid \
 scripts/images/eks-node/mulga-ebs-byid.sh:/usr/local/sbin/mulga-ebs-byid \
 scripts/images/eks-node/mulga-eks-provider-id.initd:/etc/init.d/mulga-eks-provider-id \
 scripts/images/eks-node/mulga-eks-provider-id.sh:/usr/local/sbin/mulga-eks-provider-id \
 scripts/images/eks-node/mulga-mgmt-net.initd:/etc/init.d/mulga-mgmt-net \
 scripts/images/eks-node/mulga-mgmt-net.sh:/usr/local/sbin/mulga-mgmt-net \
+scripts/images/eks-node/mulga-vpc-mtu.initd:/etc/init.d/mulga-vpc-mtu \
+scripts/images/eks-node/mulga-vpc-mtu.sh:/usr/local/sbin/mulga-vpc-mtu \
+scripts/images/eks-node/mulga-eks-k3s-recovery.initd:/etc/init.d/mulga-eks-k3s-recovery \
+scripts/images/eks-node/mulga-eks-k3s-recovery.sh:/usr/local/sbin/mulga-eks-k3s-recovery \
 scripts/images/eks-node/addons/spinifex-noop/0.1.0/noop.yaml:/usr/share/spinifex-eks/addons/spinifex-noop/0.1.0/noop.yaml \
 scripts/images/eks-node/addons/argocd/3.0.23/argocd.yaml:/usr/share/spinifex-eks/addons/argocd/3.0.23/argocd.yaml \
 scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/00-crds.yaml:/usr/share/spinifex-eks/addons/aws-load-balancer-controller/2.11.0/00-crds.yaml \
@@ -52,16 +62,20 @@ scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/10-controller.yaml:/usr
 scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/20-node.yaml:/usr/share/spinifex-eks/addons/aws-ebs-csi-driver/1.40.1/20-node.yaml \
 scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/30-storageclass.yaml:/usr/share/spinifex-eks/addons/aws-ebs-csi-driver/1.40.1/30-storageclass.yaml \
 scripts/images/eks-node/coredns/coredns-mulga.yaml:/usr/share/spinifex-eks/coredns/coredns-mulga.yaml \
-scripts/images/eks-node/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot"
+scripts/images/eks-node/cloud-init-ec2.cfg:/etc/cloud/cloud.cfg.d/90-spinifex-ec2-imds.cfg \
+scripts/images/eks-node/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot \
+scripts/images/eks-node/mulga-eks-etcd-snapshot.sh:/etc/periodic/15min/mulga-eks-etcd-snapshot"
 
 # crond (etcd snapshot), the role selector, and the role-agnostic node services
-# (EBS by-id, provider-id, mgmt NIC) at bake. mulga-mgmt-net configures the
-# system-VM NICs from fw_cfg before cloud-init/k3s — DHCP on the primary data
-# NIC so the Ec2 datasource reaches IMDS, static on mgmt0 (no-op without a
-# netcfg blob). The selector adds the role-specific services
+# (EBS by-id, provider-id) into the default runlevel at bake. mulga-mgmt-net is
+# NOT here: setup.sh adds it to the boot runlevel before cloud-init-local, since
+# it must DHCP the data NIC so the init-local Ec2 crawl reaches IMDS; a default
+# runlevel entry always runs after cloud-init-local (boot) and is too late.
+# cloud-init-ec2.cfg keeps a generous Ec2 max_wait as a safety net for host
+# br-imds datapath lag. The selector adds the role-specific services
 # (k3s/eks-token-webhook/k3s-first-boot/state-report or k3s-agent) to the
 # default runlevel on first boot.
-ENABLE_SERVICES="crond eks-node-role mulga-ebs-byid mulga-eks-provider-id mulga-mgmt-net"
+ENABLE_SERVICES="crond eks-node-role mulga-ebs-byid mulga-eks-provider-id mulga-vpc-mtu"
 
 SETUP_SCRIPT="scripts/images/eks-node/setup.sh"
 

scripts/images/eks-node/mulga-eks-addon-sync.sh

@@ -25,10 +25,10 @@ set -a
 . "${ENVFILE}"
 set +a
 
+# EKS_ACCESS_KEY/EKS_SECRET_KEY are intentionally not required: when absent,
+# eks-gateway-{fetch,publish} sign with IMDS instance-role creds via the SDK chain.
 : "${EKS_GATEWAY_URL:?}"
 : "${EKS_ADDON_GATEWAY_URL:?}"
-: "${EKS_ACCESS_KEY:?}"
-: "${EKS_SECRET_KEY:?}"
 : "${EKS_ACCOUNT_ID:?}"
 : "${EKS_CLUSTER_NAME:?}"
 
@@ -46,6 +46,15 @@ RENDER_PREFIX=spinifex-addon-
 # re-apply the addon every tick).
 WEBHOOK_CERT_DIR=/var/lib/spinifex-eks/webhook-certs
 
+# Grace before the gvks readiness gate falls back to a workload-rollout check.
+# A large bundle (e.g. the 1.45 MB argocd manifest) applies and runs but K3s'
+# auto-deploy controller never records its addon.k3s.cattle.io/gvks annotation,
+# so the gvks-only gate would report applied forever. After this many seconds
+# with the annotation still empty, readiness is decided by the bundle's
+# workloads being rolled out instead. Kept generous so fast addons settle on the
+# gvks path and slow-rolling workloads are not called ready prematurely.
+ADDON_READY_GRACE=${ADDON_READY_GRACE:-90}
+
 # log mirrors to syslog (on-VM /var/log) and, best-effort, to the serial
 # console — the host captures ttyS0 to a per-instance log, so this is the only
 # delivery diagnostics channel reachable from the host (the VM has no
@@ -263,6 +272,53 @@ addon_gvks() {
         | grep -v '^$' | head -1 || true
 }
 
+# addon_apply_age ADDON — seconds since this process first saw ADDON delivered
+# but not yet gvks-ready, recorded in a per-addon sentinel. A gvks-ready tick (or
+# an unstage) clears the sentinel via addon_apply_clear so a later re-apply
+# restarts the grace window.
+addon_apply_age() {
+    _s=/tmp/.addon-firstapply-"$1"
+    _now=$(date +%s)
+    if [ -f "${_s}" ]; then
+        _t=$(cat "${_s}" 2>/dev/null || echo "${_now}")
+    else
+        _t=${_now}
+        echo "${_now}" > "${_s}"
+    fi
+    echo $(( _now - _t ))
+}
+
+addon_apply_clear() {
+    rm -f /tmp/.addon-firstapply-"$1"
+}
+
+# addon_workloads_ready ADDON — fallback readiness when the gvks annotation never
+# lands: true only when the rendered bundle carries at least one workload and
+# every Deployment/StatefulSet/DaemonSet in it is fully rolled out. Reads the live
+# status of exactly the objects this agent applied (kubectl get -f on the rendered
+# file), so it is addon-agnostic. A bundle whose CRDs are not yet established makes
+# kubectl get -f error out, yielding "not ready" — safe, the next tick retries.
+addon_workloads_ready() {
+    _wf="${DEPLOY_DIR}/${RENDER_PREFIX}$1.yaml"
+    [ -e "${_wf}" ] || return 1
+    kubectl get -f "${_wf}" -o json --ignore-not-found 2>/dev/null \
+        | jq -e '
+            [ .items[]? ]
+            | map(select(.kind == "Deployment" or .kind == "StatefulSet" or .kind == "DaemonSet"))
+            | (length > 0)
+              and all(.[];
+                (.kind == "Deployment"
+                  and (.status.observedGeneration // 0) >= (.metadata.generation // 0)
+                  and (.status.availableReplicas // 0) >= (.spec.replicas // 1))
+                or (.kind == "StatefulSet"
+                  and (.status.observedGeneration // 0) >= (.metadata.generation // 0)
+                  and (.status.readyReplicas // 0) >= (.spec.replicas // 1))
+                or (.kind == "DaemonSet"
+                  and (.status.desiredNumberScheduled // 0) > 0
+                  and (.status.numberReady // 0) >= (.status.desiredNumberScheduled // 0)))
+        ' >/dev/null 2>&1
+}
+
 # diag_addon ADDON — one-shot console dump of the live K3s state for a stuck
 # add-on, so a delivery that never reaches ready is diagnosable from the
 # host-captured serial console (the VM is otherwise unreachable). Fires once per
@@ -304,12 +360,21 @@ sync_once() {
         if render_addon "${_addon}" "${_version}" "${_role}"; then
             _gvks=$(addon_gvks "${_addon}")
             if [ -n "${_gvks}" ]; then
+                addon_apply_clear "${_addon}"
                 log "addon ${_addon}/${_version}: applied + ready (gvks=${_gvks})"
                 report ready "${_addon}" "${_version}"
             else
-                log "addon ${_addon}/${_version}: applied, not ready yet (no K3s Addon CR for ${RENDER_PREFIX}${_addon}.yaml with populated .status.gvks)"
-                diag_addon "${_addon}"
-                report applied "${_addon}" "${_version}"
+                # gvks absent: fall back to workload rollout once past the grace
+                # window, so a large bundle K3s never annotates still reaches ready.
+                _age=$(addon_apply_age "${_addon}")
+                if [ "${_age}" -ge "${ADDON_READY_GRACE}" ] && addon_workloads_ready "${_addon}"; then
+                    log "addon ${_addon}/${_version}: applied + ready (workloads rolled out; gvks absent after ${_age}s)"
+                    report ready "${_addon}" "${_version}"
+                else
+                    log "addon ${_addon}/${_version}: applied, not ready yet (gvks empty, age=${_age}s)"
+                    diag_addon "${_addon}"
+                    report applied "${_addon}" "${_version}"
+                fi
             fi
         fi
     done < "${_staged}"
@@ -325,6 +390,7 @@ sync_once() {
         _name=${_name%.yaml}
         if ! printf '%s\n' "${_names}" | grep -qx "${_name}"; then
             log "unstaging addon ${_name} (no longer staged)"
+            addon_apply_clear "${_name}"
             # Copy the manifest aside, drop the original so K3s stops tracking it
             # (and never re-applies), then delete the objects it rendered.
             # --wait=false so namespace finalizers don't stall the sync loop.

scripts/images/eks-node/mulga-eks-etcd-snapshot.sh

@@ -1,24 +1,38 @@
 #!/bin/sh
 set -eu
 
-# mulga-eks-etcd-snapshot — nightly K3s embedded etcd snapshot to predastore.
-# Run from /etc/periodic/daily (Alpine crond).
-# K3s has a built-in `k3s etcd-snapshot` command that handles the snapshot
-# semantics; this wrapper exposes it on cron and uploads to the system-bucket
-# (see eks-v1.md Q6 + Q17).
+# mulga-eks-etcd-snapshot — K3s embedded etcd snapshot to predastore, on two
+# cron cadences. The same script is installed into two Alpine crond periodic
+# dirs and picks its cadence tier from the dir it was invoked from:
+#   /etc/periodic/15min/mulga-eks-etcd-snapshot -> tier "frequent" (RPO window)
+#   /etc/periodic/daily/mulga-eks-etcd-snapshot -> tier "daily"    (nightly)
+# Nightly alone is too coarse a recovery point for a young cluster; the frequent
+# tier bounds data loss to the cron interval. K3s has a built-in
+# `k3s etcd-snapshot` command that handles the snapshot semantics; this wrapper
+# exposes it on cron and uploads to the system-bucket (see eks-v1.md Q6 + Q17).
 #
-# Bucket layout (Q17):
-#   eks-backups-system/{accountID}/{clusterName}/etcd-{timestamp}.snap
-# Retention: managed via S3 lifecycle policy on the bucket; this script does
-# not prune.
+# Bucket layout (Q17): the tier is a key-prefix segment so a predastore S3
+# lifecycle rule can expire the frequent rolling window shallow while keeping
+# the nightly copies deep:
+#   eks-backups-system/{accountID}/{clusterName}/etcd-{tier}-{timestamp}.snap
+# This script does not prune; retention is the bucket lifecycle policy's job.
+
+# Cadence tier from the invoking crond dir; TIER overrides it (unit test).
+if [ -z "${TIER:-}" ]; then
+    case "$0" in
+        */15min/*) TIER=frequent ;;
+        *)         TIER=daily ;;
+    esac
+fi
 
 # Worker nodes share the unified eks-node AMI but have no etcd/SQLite datastore
 # to snapshot. The role file is written once by eks-node-role at first boot.
-if [ "$(cat /etc/spinifex-eks/role 2>/dev/null)" = "agent" ]; then
+ROLE_FILE=${ROLE_FILE:-/etc/spinifex-eks/role}
+if [ "$(cat "${ROLE_FILE}" 2>/dev/null)" = "agent" ]; then
     exit 0
 fi
 
-ENVFILE=/etc/spinifex-eks/etcd-snapshot.env
+ENVFILE=${ENVFILE:-/etc/spinifex-eks/etcd-snapshot.env}
 [ -f "${ENVFILE}" ] || { logger -t mulga-eks-etcd-snapshot "${ENVFILE} missing"; exit 0; }
 # shellcheck disable=SC1090
 . "${ENVFILE}"
@@ -29,13 +43,16 @@ ENVFILE=/etc/spinifex-eks/etcd-snapshot.env
 : "${SPINIFEX_PREDASTORE_AKID:?}"
 : "${SPINIFEX_PREDASTORE_SECRET:?}"
 
-SNAPSHOT_DIR=/var/lib/rancher/k3s/server/db/snapshots
+K3S_BIN=${K3S_BIN:-/usr/local/bin/k3s}
+SNAPSHOT_DIR=${SNAPSHOT_DIR:-/var/lib/rancher/k3s/server/db/snapshots}
 TS=$(date -u +%Y%m%dT%H%M%SZ)
-NAME="etcd-${TS}"
+# Tier in the name keeps the two cadences from colliding on an identical second
+# (both dirs can fire at 02:00) and prefix-filters cleanly for S3 lifecycle.
+NAME="etcd-${TIER}-${TS}"
 BUCKET="eks-backups-system"
 KEY="${EKS_ACCOUNT_ID}/${EKS_CLUSTER_NAME}/${NAME}.snap"
 
-/usr/local/bin/k3s etcd-snapshot save --name "${NAME}" 2>&1 | logger -t mulga-eks-etcd-snapshot
+"${K3S_BIN}" etcd-snapshot save --name "${NAME}" 2>&1 | logger -t mulga-eks-etcd-snapshot
 
 SNAPSHOT_FILE=$(ls -t "${SNAPSHOT_DIR}/${NAME}"* 2>/dev/null | head -1 || true)
 if [ -z "${SNAPSHOT_FILE}" ]; then

scripts/images/eks-node/mulga-eks-etcd-snapshot_test.sh

@@ -0,0 +1,110 @@
+#!/bin/sh
+# Self-contained POSIX test for mulga-eks-etcd-snapshot.sh cadence + upload. No
+# k3s/predastore/root: k3s, curl and logger are stubbed on PATH; the role file,
+# ENVFILE, SNAPSHOT_DIR and K3S_BIN are redirected into a temp dir via the
+# script's env knobs. The k3s stub fabricates the snapshot file the script then
+# uploads; the curl stub records the request line so we can assert the key.
+#
+# Run: sh scripts/images/eks-node/mulga-eks-etcd-snapshot_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+SCRIPT="${SCRIPT_DIR}/mulga-eks-etcd-snapshot.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+STUBBIN="${WORK}/bin"
+mkdir -p "${STUBBIN}"
+
+# k3s stub: on `etcd-snapshot save --name <NAME>` fabricate the snapshot file the
+# real k3s would drop under SNAPSHOT_DIR, and record that it was invoked.
+cat > "${STUBBIN}/k3s" <<'EOF'
+#!/bin/sh
+echo "k3s $*" >> "${K3S_CALLS}"
+name=
+while [ $# -gt 0 ]; do
+    case "$1" in --name) name="$2"; shift 2 ;; *) shift ;; esac
+done
+[ -n "${name}" ] && { mkdir -p "${SNAPSHOT_DIR}"; echo snap > "${SNAPSHOT_DIR}/${name}.zip"; }
+:
+EOF
+
+# curl stub: record the full argument line (carries --upload-file + URL).
+cat > "${STUBBIN}/curl" <<'EOF'
+#!/bin/sh
+echo "curl $*" >> "${CURL_CALLS}"
+EOF
+
+# logger stub: swallow.
+cat > "${STUBBIN}/logger" <<'EOF'
+#!/bin/sh
+:
+EOF
+
+chmod +x "${STUBBIN}"/*
+PATH="${STUBBIN}:${PATH}"
+export PATH
+
+ENVFILE="${WORK}/etcd-snapshot.env"
+cat > "${ENVFILE}" <<'EOF'
+EKS_ACCOUNT_ID=000000000001
+EKS_CLUSTER_NAME=demo3
+SPINIFEX_PREDASTORE_ENDPOINT=https://pred.invalid:8443
+SPINIFEX_PREDASTORE_AKID=AKIDTEST
+SPINIFEX_PREDASTORE_SECRET=SECRETTEST
+EOF
+
+FAILS=0
+fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
+pass() { echo "ok: $*"; }
+
+# Symlinks whose parent dir names the cadence — invoking them gives the script
+# the real $0 it sniffs in production, exercising the crond-dir tier selection.
+mkdir -p "${WORK}/15min" "${WORK}/daily"
+ln -sf "${SCRIPT}" "${WORK}/15min/mulga-eks-etcd-snapshot"
+ln -sf "${SCRIPT}" "${WORK}/daily/mulga-eks-etcd-snapshot"
+
+# run <role> <invocation-path>: resets the capture files + knobs and runs the
+# script via the given symlink so $0 (the cadence source) is authentic.
+run() {
+    _role=$1; _path=$2
+    K3S_CALLS="${WORK}/k3s.calls"; CURL_CALLS="${WORK}/curl.calls"
+    SNAPSHOT_DIR="${WORK}/snapshots"; ROLE_FILE="${WORK}/role"; K3S_BIN=k3s
+    : > "${K3S_CALLS}"; : > "${CURL_CALLS}"; rm -rf "${SNAPSHOT_DIR}"
+    printf '%s' "${_role}" > "${ROLE_FILE}"
+    export K3S_CALLS CURL_CALLS SNAPSHOT_DIR ROLE_FILE K3S_BIN ENVFILE
+    unset TIER 2>/dev/null || true
+    sh "${_path}" </dev/null || fail "${_path}: non-zero exit"
+}
+
+# --- Case 1: agent role -> no snapshot, no upload ---
+run agent "${WORK}/15min/mulga-eks-etcd-snapshot"
+[ -s "${WORK}/k3s.calls" ] && fail "agent: must not snapshot" || pass "agent: no k3s call"
+[ -s "${WORK}/curl.calls" ] && fail "agent: must not upload" || pass "agent: no upload"
+
+# --- Case 2: server, 15min dir -> frequent tier upload key ---
+run server "${WORK}/15min/mulga-eks-etcd-snapshot"
+grep -q 'etcd-snapshot save --name etcd-frequent-' "${WORK}/k3s.calls" \
+    && pass "frequent: k3s snapshot named for tier" || fail "frequent: k3s not called for tier: $(cat "${WORK}/k3s.calls")"
+grep -q '/eks-backups-system/000000000001/demo3/etcd-frequent-.*\.snap' "${WORK}/curl.calls" \
+    && pass "frequent: upload key carries tier + identity" || fail "frequent: upload key wrong: $(cat "${WORK}/curl.calls")"
+ls "${WORK}/snapshots"/etcd-frequent-* >/dev/null 2>&1 \
+    && fail "frequent: local snapshot must be pruned after upload" || pass "frequent: local snapshot pruned"
+
+# --- Case 3: server, daily dir -> daily tier upload key ---
+run server "${WORK}/daily/mulga-eks-etcd-snapshot"
+grep -q '/eks-backups-system/000000000001/demo3/etcd-daily-.*\.snap' "${WORK}/curl.calls" \
+    && pass "daily: cadence sniffed from periodic dir" || fail "daily: sniff wrong: $(cat "${WORK}/curl.calls")"
+
+# --- Case 4: missing ENVFILE -> quiet no-op, no upload ---
+ENVFILE_SAVE="${ENVFILE}"; ENVFILE="${WORK}/missing.env"
+run server "${WORK}/15min/mulga-eks-etcd-snapshot"
+[ -s "${WORK}/curl.calls" ] && fail "no-env: must not upload" || pass "no-env: no upload without creds"
+ENVFILE="${ENVFILE_SAVE}"
+
+if [ "${FAILS}" -eq 0 ]; then
+    echo "PASS: all mulga-eks-etcd-snapshot cases"
+    exit 0
+fi
+echo "FAILED: ${FAILS} case(s)"
+exit 1