refactor(ecr): delete the in-memory MetaStore and run its tests against JetStream KV (#1474)

* feat(admin): non-finalizing DeleteAccount exercises the quiesced EC2 platform kinds

A non-finalizing DeleteAccount quiesces the account, records the S9 evaluation, cleans only the key-pair, placement-group and launch-template kinds it recorded as quiesced, then re-reads the owner inventory and accounts for what remains. The job ends EXERCISED with the account still TERMINATING; a later ordinary delete takes the same operation over.

Only an exercise records a quiesced scope, so ordinary deletes still record none. The cleanup capability gains a kinds selector, awsgw binds the in-process owners, and the instance reference reader fails closed.

* refactor(admin): name the takeover of a finished exercise exercise_finished

.github/scripts/wait-for-memory.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Usage: wait-for-memory.sh <MiB>
+#
+# Waits until the runner's MemAvailable reaches <MiB>, then returns. Runners on
+# one host share its RAM, and a neighbour's build holds several GiB for a few
+# minutes, so one sample taken during it is not a verdict on the host.
+# Gives up after MEM_WAIT_TIMEOUT_S and names the biggest processes.
+#
+# Env: MEM_WAIT_TIMEOUT_S (900), MEM_WAIT_POLL_S (30), MEMINFO (/proc/meminfo).
+set -uo pipefail
+
+want_mb="${1:?usage: $0 <MiB>}"
+timeout_s="${MEM_WAIT_TIMEOUT_S:-900}"
+poll_s="${MEM_WAIT_POLL_S:-30}"
+meminfo="${MEMINFO:-/proc/meminfo}"
+
+avail_mb() { awk '/^MemAvailable:/ {print int($2 / 1024)}' "$meminfo"; }
+
+start=$SECONDS
+while :; do
+  have=$(avail_mb)
+  if [ -n "$have" ] && [ "$have" -ge "$want_mb" ]; then
+    echo "memory: ${have} MiB available, need ${want_mb} MiB"
+    exit 0
+  fi
+  waited=$(( SECONDS - start ))
+  if [ "$waited" -ge "$timeout_s" ]; then
+    echo "::error::need ${want_mb} MiB available, have ${have:-unknown} MiB after waiting ${waited}s"
+    ps -eo rss,pid,args --sort=-rss | head -11 | cut -c1-200
+    exit 1
+  fi
+  echo "memory: ${have:-unknown} MiB available, need ${want_mb} MiB; waiting (${waited}s of ${timeout_s}s)"
+  sleep "$poll_s"
+done

.github/scripts/wait-for-memory_test.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+# wait-for-memory_test.sh — drives wait-for-memory.sh against a fake meminfo
+# that a background writer changes mid-wait, so each verdict is checked without
+# a loaded host.
+set -eu
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+WAIT="$SCRIPT_DIR/wait-for-memory.sh"
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+meminfo() { printf 'MemTotal:       65536000 kB\nMemAvailable:   %s kB\n' "$(( $1 * 1024 ))" > "$TMP/meminfo"; }
+MEMINFO="$TMP/meminfo"
+MEM_WAIT_POLL_S=1
+export MEMINFO MEM_WAIT_POLL_S
+
+meminfo 30000
+if MEM_WAIT_TIMEOUT_S=5 bash "$WAIT" 26624 > "$TMP/out" 2>&1; then
+  pass "enough memory returns at once"
+else
+  fail "enough memory returns at once: $(cat "$TMP/out")"
+fi
+
+meminfo 21577
+( sleep 2; meminfo 30000 ) &
+if MEM_WAIT_TIMEOUT_S=20 bash "$WAIT" 26624 > "$TMP/out" 2>&1 && grep -q waiting "$TMP/out"; then
+  pass "memory freed mid-wait is admitted"
+else
+  fail "memory freed mid-wait is admitted: $(cat "$TMP/out")"
+fi
+wait
+
+meminfo 21577
+if MEM_WAIT_TIMEOUT_S=2 bash "$WAIT" 26624 > "$TMP/out" 2>&1; then
+  fail "memory that never frees fails: exited 0"
+elif grep -q '::error::need 26624 MiB available, have 21577 MiB' "$TMP/out" && grep -q 'RSS' "$TMP/out"; then
+  pass "memory that never frees fails and names the biggest processes"
+else
+  fail "memory that never frees fails and names the biggest processes: $(cat "$TMP/out")"
+fi
+
+[ "$fails" -eq 0 ] || { echo "wait-for-memory: $fails failed"; exit 1; }
+echo "wait-for-memory: all passed"

.github/workflows/e2e-nightly.yml

@@ -1057,9 +1057,6 @@ jobs:
             { echo "::error::br-wan not permitted in /etc/qemu/bridge.conf"; exit 1; }
           command -v sshpass >/dev/null ||
             { echo "::error::sshpass not installed"; exit 1; }
-          free_mb=$(free -m | awk '/^Mem:/ {print $7}')
-          [ "$free_mb" -ge 8192 ] ||
-            { echo "::error::need 8 GiB available, have ${free_mb} MiB"; exit 1; }
           # A 40G image on tmpfs competes with the 8 GiB the VM itself needs.
           fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
           [ "$fstype" != tmpfs ] ||
@@ -1090,6 +1087,10 @@ jobs:
               ssh-keygen -t ed25519 -N "" -f "$HOME/.ssh/$k" -C "$k-ci"
           done
 
+      # The node's 8 GiB, claimed when the VM boots rather than at preflight.
+      - name: Wait for memory
+        run: mulga/spinifex/.github/scripts/wait-for-memory.sh 8192
+
       # Boots the ISO, installs unattended, reads the lease off the serial
       # console, runs the in-guest smoke test, and installs the key for the
       # next step.
@@ -1301,15 +1302,10 @@ jobs:
           ref: ${{ steps.siblings.outputs.bluebottle }}
           path: mulga/bluebottle
 
-      # 3 x 8 GiB plus 9 sparse 40G disks, before the instances the cluster
-      # launches on top.
       - name: Preflight
         run: |
           set -euo pipefail
           mkdir -p "$TMPDIR"
-          free_mb=$(free -m | awk '/^Mem:/ {print $7}')
-          [ "$free_mb" -ge 26624 ] ||
-            { echo "::error::need 26 GiB available, have ${free_mb} MiB"; exit 1; }
           fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
           [ "$fstype" != tmpfs ] ||
             { echo "::error::$TMPDIR is tmpfs — nine node disks would be in RAM"; exit 1; }
@@ -1332,6 +1328,11 @@ jobs:
       # formation port. Both land with mulga-yxtda and the bridged WAN mode;
       # until then this cell proves the cluster forms and survives, not that a
       # customer can use it.
+      # 3 x 8 GiB plus headroom for the instances the cluster launches on top,
+      # claimed when the nodes boot rather than at preflight.
+      - name: Wait for memory
+        run: mulga/spinifex/.github/scripts/wait-for-memory.sh 26624
+
       - name: Install, form and verify the cluster
         working-directory: mulga/scripts/iso-builder
         run: |

.gitignore

@@ -24,6 +24,9 @@ build/microvm/qboot.LICENSE
 build/microvm/sqs-worker/vmlinuz
 build/microvm/sqs-worker/initramfs.cpio.gz
 build/microvm/sqs-worker/.stage.*/
+build/microvm/lambda-runtime/vmlinuz
+build/microvm/lambda-runtime/initramfs.cpio.gz
+build/microvm/lambda-runtime/.stage.*/
 build/qboot/
 !spinifex/runtime/roles/spinifexui/frontend/dist/
 tmp/

Makefile

@@ -169,7 +169,7 @@ go_local_srcs = $(shell GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go list
 # worker as separate processes. A separate artifact from the LB image above.
 SQS_WORKER_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/sqs-worker
 SQS_WORKER_MICROVM_ARTIFACTS := $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz $(SQS_WORKER_MICROVM_OUT_DIR)/initramfs.cpio.gz
-SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_WORKER_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-store sqs-worker-ready sqs-worker-stop) bin/nats-server bin/sqs-worker
+SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_WORKER_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-store sqs-worker-ready sqs-worker-stop) bin/nats-server bin/sqs-worker
 
 $(SQS_WORKER_MICROVM_ARTIFACTS) &: $(SQS_WORKER_MICROVM_INPUTS)
 	./scripts/build-sqs-store-microvm-image.sh
@@ -180,6 +180,20 @@ bin/sqs-worker: go.mod go.sum $$(call go_local_srcs,./cmd/sqs-worker)
 
 build-sqs-worker-microvm-image: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Build the SQS microVM kernel and initramfs
 
+# The Lambda runtime microVM image. Built on demand; not shipped or installed.
+LAMBDA_RUNTIME_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/lambda-runtime
+LAMBDA_RUNTIME_MICROVM_ARTIFACTS := $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/vmlinuz $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/initramfs.cpio.gz
+LAMBDA_RUNTIME_MICROVM_INPUTS := scripts/build-lambda-runtime-microvm-image.sh scripts/lib/microvm-image.sh $(addprefix $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/,init.sh inittab run-agent) bin/lambda-agent
+
+$(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) &: $(LAMBDA_RUNTIME_MICROVM_INPUTS)
+	./scripts/build-lambda-runtime-microvm-image.sh
+
+bin/lambda-agent: go.mod go.sum $$(call go_local_srcs,./cmd/lambda-agent)
+	@mkdir -p bin
+	GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/lambda-agent
+
+build-lambda-runtime-microvm-image: $(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) ## Build the Lambda runtime microVM kernel and initramfs (not shipped)
+
 # qboot is the microvm firmware. The source tarball ships with each release
 # because qboot is GPL-2.0.
 QBOOT_COMMIT := $(shell sed -n 's/^QBOOT_COMMIT="\(.*\)"$$/\1/p' scripts/build-qboot.sh)
@@ -241,7 +255,8 @@ test-build-scripts:
 		echo "-- $$t"; \
 		sh "$$t" || exit 1; \
 	done
-	shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh .github/scripts/hypervisor-disk-preflight_test.sh
+	shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh .github/scripts/hypervisor-disk-preflight_test.sh \
+		.github/scripts/wait-for-memory.sh .github/scripts/wait-for-memory_test.sh
 	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
 	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
 	shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
@@ -312,6 +327,7 @@ test-cover:
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
 	$(_Q)LOG_IGNORE=1 SPINIFEX_SQS_R3=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -522,7 +538,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
+.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -113,7 +113,7 @@ aws ec2 describe-instance-types
 ## Web Console
 
 <p align="center">
-  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
+  <img src=".github/assets/dashboard.webp" alt="Spinifex web console dashboard" width="900">
 </p>
 
 Spinifex ships with a built-in web console for your instances, volumes, buckets, networks, and IAM. See [Launching the Web UI](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui).

build/microvm/lambda-runtime/init.sh

@@ -0,0 +1,68 @@
+#!/bin/sh
+mount -t proc  proc  /proc
+mount -t sysfs sys   /sys
+mount -t devtmpfs dev /dev
+
+# modprobe needs a usable modules.dep; insmod by explicit path does not, so
+# fall back to it. Handles .ko and .ko.gz.
+load_mod() {
+    modprobe "$1" 2>/dev/null && return 0
+    for ko in $(find /lib/modules -name "$1.ko" -o -name "$1.ko.gz" 2>/dev/null); do
+        insmod "$ko" 2>/dev/null && return 0
+        case "$ko" in
+            *.gz) gunzip -c "$ko" >/tmp/mod.ko 2>/dev/null \
+                  && insmod /tmp/mod.ko 2>/dev/null && return 0 ;;
+        esac
+    done
+    return 1
+}
+# lambda-agent reads its identity from fw_cfg and unloads qemu_fw_cfg before
+# anything else runs. virtio_net needs net_failover and failover, and the
+# insmod fallback resolves no dependencies, so load the chain bottom-up.
+load_mod qemu_fw_cfg || echo "[init] qemu_fw_cfg did not load" >/dev/console
+load_mod failover || true
+load_mod net_failover || true
+load_mod virtio_net || echo "[init] virtio_net did not load" >/dev/console
+
+ip link set lo up
+
+# The guest's addresses come from the same netcfg entry as other system VMs.
+NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
+i=0
+while [ "$i" -lt 25 ] && [ ! -f "$NETCFG" ]; do
+    sleep 0.04
+    i=$((i + 1))
+done
+if [ -f "$NETCFG" ]; then
+    # shellcheck source=/dev/null
+    . "$NETCFG"
+    for n in 0 1 2 3 4 5; do
+        eval mac=\$NIC${n}_MAC
+        [ -z "$mac" ] && continue
+        iface=""
+        for _ in 1 2 3 4 5; do
+            iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :)
+            [ -n "$iface" ] && break
+            sleep 0.02
+        done
+        if [ -z "$iface" ]; then
+            echo "[init] no interface for MAC $mac (NIC$n)" >/dev/console
+            continue
+        fi
+        cidr="" gw="" default="" rdst="" rvia=""
+        eval cidr=\$NIC${n}_CIDR
+        eval gw=\$NIC${n}_GW
+        eval default=\$NIC${n}_DEFAULT
+        eval rdst=\$NIC${n}_ROUTE_DST
+        eval rvia=\$NIC${n}_ROUTE_VIA
+        ip link set "$iface" up
+        [ -n "$cidr" ] && ip addr add "$cidr" dev "$iface"
+        [ "$default" = "1" ] && [ -n "$gw" ] && ip route add default via "$gw" dev "$iface"
+        [ -n "$rdst" ] && [ -n "$rvia" ] && ip route add "$rdst" via "$rvia" dev "$iface"
+    done
+else
+    # lambda-agent then fails to reach the host and the guest powers off.
+    echo "[init] fw_cfg netcfg not available" >/dev/console
+fi
+
+exec /sbin/init

build/microvm/lambda-runtime/inittab

@@ -0,0 +1,6 @@
+# The power button loads here, not in /init: busybox init has its SIGUSR2
+# (poweroff) handler installed before sysinit runs, so a press can never reach
+# the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
+::sysinit:/sbin/modprobe tiny_power_button power_signal=12
+::once:/usr/local/lib/lambda-runtime/run-agent
+::ctrlaltdel:/sbin/reboot

build/microvm/lambda-runtime/run-agent

@@ -0,0 +1,6 @@
+#!/bin/sh
+# Runs the Lambda guest agent, then powers the guest off whatever its status.
+/usr/local/bin/lambda-agent >/dev/console 2>&1
+status=$?
+echo "[run-agent] lambda-agent exited with status $status" >/dev/console
+poweroff -f

build/systemd/spinifex-daemon.service

@@ -1,4 +1,4 @@
-# spinifex-unit-version: 5
+# spinifex-unit-version: 6
 [Unit]
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
@@ -15,6 +15,7 @@ Slice=spinifex-system.slice
 User=spinifex-daemon
 Group=spinifex
 ExecStartPre=/usr/local/bin/spx admin hostnet check
+ExecStartPre=-+/usr/local/bin/spx admin node-facts collect
 ExecStart=/usr/local/bin/spx service spinifex start
 Restart=on-failure
 RestartSec=5

cmd/lambda-agent/connect.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+)
+
+// Exit statuses: the guest refused to start, the control connection failed,
+// or the host closed it.
+const (
+	exitRefused = 2
+	exitLink    = 1
+	exitClosed  = 0
+)
+
+type startup struct {
+	fwCfgDir  string
+	fwCfgRoot string
+	unload    func() error
+}
+
+func defaultStartup() startup {
+	return startup{fwCfgDir: hostlink.FwCfgDir, fwCfgRoot: hostlink.FwCfgRoot, unload: hostlink.UnloadFwCfg}
+}
+
+// connect loads the identity, seals fw_cfg whether or not the load succeeded,
+// then dials the host. Under TLS 1.3 a host refusal is seen only on first read.
+func (s startup) connect(ctx context.Context) (*tls.Conn, int, error) {
+	id, loadErr := hostlink.LoadIdentity(s.fwCfgDir)
+	if err := hostlink.SealFwCfg(s.fwCfgRoot, s.unload); err != nil {
+		return nil, exitRefused, errors.Join(loadErr, err)
+	}
+	if loadErr != nil {
+		return nil, exitRefused, loadErr
+	}
+	conn, err := hostlink.Dial(ctx, id)
+	if err != nil {
+		if ctx.Err() != nil {
+			err = fmt.Errorf("%w: %w", err, context.Cause(ctx))
+		}
+		return nil, exitLink, err
+	}
+	return conn, 0, nil
+}

cmd/lambda-agent/helpers_test.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/guestconn"
+	"github.com/stretchr/testify/require"
+)
+
+const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-1"
+
+// guest is a fake fw_cfg tree; unload removes it as unloading the module would.
+type guest struct {
+	root     string
+	unloads  int
+	sealedAt chan struct{}
+}
+
+func newGuest(t *testing.T, entries map[string][]byte) *guest {
+	t.Helper()
+	g := &guest{root: filepath.Join(t.TempDir(), "qemu_fw_cfg"), sealedAt: make(chan struct{}, 1)}
+	for name, b := range entries {
+		dir := filepath.Join(g.root, "by_name", lambdav1.FwCfgPrefix, name)
+		require.NoError(t, os.MkdirAll(dir, 0o700))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "raw"), b, 0o600))
+	}
+	return g
+}
+
+func (g *guest) startup() startup {
+	return startup{
+		fwCfgDir:  filepath.Join(g.root, "by_name", lambdav1.FwCfgPrefix),
+		fwCfgRoot: g.root,
+		unload: func() error {
+			g.unloads++
+			g.sealedAt <- struct{}{}
+			return os.RemoveAll(g.root)
+		},
+	}
+}
+
+type host struct {
+	ln   *net.TCPListener
+	gen  *guestconn.Generation
+	conn chan accepted
+}
+
+func newHost(t *testing.T) *host {
+	t.Helper()
+	ln, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)})
+	require.NoError(t, err)
+	t.Cleanup(func() { ln.Close() })
+	gen, err := guestconn.NewGeneration(scope, time.Hour)
+	require.NoError(t, err)
+	return &host{ln: ln, gen: gen, conn: make(chan accepted, 1)}
+}
+
+func (h *host) entries() map[string][]byte { return h.gen.GuestEntries(h.ln.Addr().String()) }
+
+type accepted struct {
+	conn *tls.Conn
+	err  error
+}
+
+// accept runs the host end until ctx ends and reports what it accepted.
+func (h *host) accept(ctx context.Context) {
+	go func() {
+		c, err := h.gen.AcceptGuest(ctx, h.ln, 2*time.Second)
+		h.conn <- accepted{c, err}
+	}()
+}
+
+// guest waits for the host to authenticate the guest.
+func (h *host) guest(t *testing.T) *tls.Conn {
+	t.Helper()
+	a := wait(t, h.conn, "host to authenticate the guest")
+	require.NoError(t, a.err)
+	return a.conn
+}
+
+// wait bounds every test wait so a stalled host or startup fails promptly.
+func wait[T any](t *testing.T, ch <-chan T, what string) T {
+	t.Helper()
+	select {
+	case v := <-ch:
+		return v
+	case <-time.After(10 * time.Second):
+		t.Fatalf("timed out waiting for %s", what)
+		panic("unreachable")
+	}
+}

cmd/lambda-agent/main.go

@@ -0,0 +1,24 @@
+// lambda-agent runs inside a Lambda guest.
+//
+// A normal build reads its per-generation identity from fw_cfg, unloads
+// fw_cfg so nothing started later can read the guest key, and opens the
+// authenticated control connection to the host. It accepts no invocations and
+// never starts the function's bootstrap: the message protocol and grant
+// fencing do not exist yet. The agent never listens for the host.
+//
+// A lambdatestlink build opens the same connection, then relays one invocation
+// over a test carrier for local proofs; the carrier is not the guest protocol.
+package main
+
+import (
+	"context"
+	"flag"
+	"os"
+)
+
+func main() { os.Exit(run()) }
+
+func run() int {
+	flag.Parse()
+	return start(context.Background())
+}

cmd/lambda-agent/package_testlink.go

@@ -0,0 +1,276 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// Dev-only code delivery for the test path: the host attaches the ZIP as a
+// read-only disk and passes its exact size and SHA-256 through fw_cfg. This
+// is not the production delivery, and these bounds are not AWS limits.
+const (
+	entryPackageSize   = "test-package-size"
+	entryPackageSHA256 = "test-package-sha256"
+
+	devMaxZipBytes    = 16 << 20
+	devMaxUnzipBytes  = 64 << 20
+	devMaxEntries     = 1024
+	packageDevice     = "/dev/vda"
+	taskTmpfsSizeOpts = "mode=0755,size=72m"
+)
+
+type packageSpec struct {
+	size   int64
+	sha256 [sha256.Size]byte
+}
+
+type unpackLimits struct {
+	entries int
+	bytes   int64
+}
+
+var devLimits = unpackLimits{entries: devMaxEntries, bytes: devMaxUnzipBytes}
+
+// readPackageSpec reads the expected size and SHA-256 from fw_cfg; it must
+// run before fw_cfg is sealed.
+func readPackageSpec(fwCfgDir string) (packageSpec, error) {
+	var spec packageSpec
+	read := func(name string) (string, error) {
+		b, err := os.ReadFile(filepath.Join(fwCfgDir, name, "raw"))
+		if err != nil {
+			return "", fmt.Errorf("read %s: %w", name, err)
+		}
+		return strings.TrimSpace(string(b)), nil
+	}
+	s, err := read(entryPackageSize)
+	if err != nil {
+		return spec, err
+	}
+	spec.size, err = strconv.ParseInt(s, 10, 64)
+	if err != nil || spec.size <= 0 || spec.size > devMaxZipBytes {
+		return spec, fmt.Errorf("package size %q outside 1..%d bytes", s, devMaxZipBytes)
+	}
+	h, err := read(entryPackageSHA256)
+	if err != nil {
+		return spec, err
+	}
+	b, err := hex.DecodeString(h)
+	if err != nil || len(b) != sha256.Size {
+		return spec, errors.New("package sha256 is not 64 hex digits")
+	}
+	copy(spec.sha256[:], b)
+	return spec, nil
+}
+
+// taskPackage puts the package under dir: a fresh nosuid,nodev tmpfs, made
+// read-only once the files are in place.
+type taskPackage struct {
+	device     string
+	dir        string
+	loadDriver func() error
+	mountDir   func(dir string) error
+	sealDir    func(dir string) error
+}
+
+func devTaskPackage() taskPackage {
+	return taskPackage{device: packageDevice, dir: taskRoot, loadDriver: loadBlockDriver, mountDir: mountTaskDir, sealDir: sealTaskDir}
+}
+
+func (p taskPackage) install(spec packageSpec) error {
+	if err := p.loadDriver(); err != nil {
+		return fmt.Errorf("load block driver: %w", err)
+	}
+	dev, err := os.Open(p.device)
+	if err != nil {
+		return fmt.Errorf("open package disk: %w", err)
+	}
+	defer dev.Close()
+	if err := p.mountDir(p.dir); err != nil {
+		return fmt.Errorf("mount %s: %w", p.dir, err)
+	}
+	if err := unpack(dev, spec, p.dir, devLimits); err != nil {
+		return err
+	}
+	if err := p.sealDir(p.dir); err != nil {
+		return fmt.Errorf("make %s read-only: %w", p.dir, err)
+	}
+	return nil
+}
+
+// unpack reads exactly spec.size bytes once, checks their SHA-256, then
+// validates every entry before writing any of them under dir.
+func unpack(r io.ReaderAt, spec packageSpec, dir string, lim unpackLimits) error {
+	buf := make([]byte, spec.size)
+	if _, err := io.ReadFull(io.NewSectionReader(r, 0, spec.size), buf); err != nil {
+		return fmt.Errorf("read package: %w", err)
+	}
+	if sha256.Sum256(buf) != spec.sha256 {
+		return errors.New("package sha256 mismatch")
+	}
+	zr, err := zip.NewReader(bytes.NewReader(buf), spec.size)
+	if err != nil {
+		return fmt.Errorf("open package: %w", err)
+	}
+	if len(zr.File) > lim.entries {
+		return fmt.Errorf("package has %d entries, limit %d", len(zr.File), lim.entries)
+	}
+	names := make([]string, len(zr.File))
+	var declared int64
+	for i, f := range zr.File {
+		name, err := entryName(f.Name)
+		if err != nil {
+			return err
+		}
+		names[i] = name
+		switch mode := f.Mode(); {
+		case mode.IsDir():
+		case mode.IsRegular():
+			// Compared against what remains, so a huge size cannot wrap the sum.
+			if f.UncompressedSize64 > uint64(lim.bytes-declared) {
+				return fmt.Errorf("%q: unpacked size over the %d-byte limit", f.Name, lim.bytes)
+			}
+			declared += int64(f.UncompressedSize64)
+		default:
+			return fmt.Errorf("%q: %s entries are refused", f.Name, mode.Type())
+		}
+	}
+
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		return err
+	}
+	defer root.Close()
+	var written int64
+	for i, f := range zr.File {
+		n, err := extract(root, names[i], f, lim.bytes-written)
+		written += n
+		if err != nil {
+			return fmt.Errorf("%q: %w", f.Name, err)
+		}
+	}
+	return nil
+}
+
+// extract writes one entry, refusing it if its actual bytes exceed budget
+// whatever its header claims.
+func extract(root *os.Root, name string, f *zip.File, budget int64) (int64, error) {
+	if f.Mode().IsDir() {
+		return 0, mkdirs(root, name)
+	}
+	if d := path.Dir(name); d != "." {
+		if err := mkdirs(root, d); err != nil {
+			return 0, err
+		}
+	}
+	perm := os.FileMode(0o644)
+	if f.Mode()&0o111 != 0 {
+		perm = 0o755
+	}
+	in, err := f.Open()
+	if err != nil {
+		return 0, err
+	}
+	defer in.Close()
+	out, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
+	if err != nil {
+		return 0, err
+	}
+	// The umask applies at create; fchmod does not, so the mode is exact.
+	if err := out.Chmod(perm); err != nil {
+		out.Close()
+		return 0, err
+	}
+	n, err := io.Copy(out, io.LimitReader(in, budget+1))
+	if cerr := out.Close(); err == nil {
+		err = cerr
+	}
+	if err == nil && n > budget {
+		err = errors.New("unpacked bytes over the limit")
+	}
+	return n, err
+}
+
+// mkdirs creates each component of dir as mode 0755 whatever the umask. An
+// existing component must be a real directory, never a symlink.
+func mkdirs(root *os.Root, dir string) error {
+	cur := ""
+	for _, part := range strings.Split(dir, "/") {
+		cur = path.Join(cur, part)
+		if err := root.Mkdir(cur, 0o755); err != nil {
+			if !errors.Is(err, fs.ErrExist) {
+				return err
+			}
+			fi, err := root.Lstat(cur)
+			if err != nil {
+				return err
+			}
+			if !fi.IsDir() {
+				return fmt.Errorf("%q exists and is not a directory", cur)
+			}
+		}
+		if err := root.Chmod(cur, 0o755); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// entryName returns a clean relative path, refusing anything that could name
+// a location outside the task root.
+func entryName(name string) (string, error) {
+	if name == "" || strings.ContainsAny(name, "\\\x00") || path.IsAbs(name) {
+		return "", fmt.Errorf("%q: entry name refused", name)
+	}
+	c := path.Clean(name)
+	if c == "." || c == ".." || strings.HasPrefix(c, "../") {
+		return "", fmt.Errorf("%q: entry name refused", name)
+	}
+	return c, nil
+}
+
+// loadBlockDriver loads virtio_blk, which the image ships but init does not
+// load, then waits briefly for the package disk to appear.
+func loadBlockDriver() error {
+	if err := exec.Command("/sbin/modprobe", "virtio_blk").Run(); err != nil {
+		var uts unix.Utsname
+		if uerr := unix.Uname(&uts); uerr != nil {
+			return uerr
+		}
+		ko := "/lib/modules/" + unix.ByteSliceToString(uts.Release[:]) + "/kernel/drivers/block/virtio_blk.ko"
+		if err := exec.Command("/sbin/insmod", ko).Run(); err != nil {
+			return fmt.Errorf("modprobe and insmod virtio_blk failed: %w", err)
+		}
+	}
+	for range 50 {
+		if _, err := os.Stat(packageDevice); err == nil {
+			return nil
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	return fmt.Errorf("%s did not appear", packageDevice)
+}
+
+func mountTaskDir(dir string) error {
+	return unix.Mount("tmpfs", dir, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, taskTmpfsSizeOpts)
+}
+
+func sealTaskDir(dir string) error {
+	return unix.Mount("", dir, "", unix.MS_REMOUNT|unix.MS_RDONLY|unix.MS_NOSUID|unix.MS_NODEV, "")
+}

cmd/lambda-agent/package_testlink_test.go

@@ -0,0 +1,209 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"hash/crc32"
+	"io/fs"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testZip builds a ZIP; an entry named bootstrap is marked executable.
+func testZip(t *testing.T, files map[string]string) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	names := make([]string, 0, len(files))
+	for n := range files {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	for _, n := range names {
+		h := &zip.FileHeader{Name: n, Method: zip.Deflate}
+		h.SetMode(0o644)
+		if n == "bootstrap" {
+			h.SetMode(0o755)
+		}
+		w, err := zw.CreateHeader(h)
+		require.NoError(t, err)
+		_, err = w.Write([]byte(files[n]))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+// rawZip writes entries with headers set by the caller, so a test can claim
+// sizes or modes the data does not have.
+func rawZip(t *testing.T, entries ...*zip.FileHeader) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for _, h := range entries {
+		w, err := zw.CreateRaw(h)
+		require.NoError(t, err)
+		_, err = w.Write([]byte("x"))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+func specFor(b []byte) packageSpec {
+	return packageSpec{size: int64(len(b)), sha256: sha256.Sum256(b)}
+}
+
+// Not parallel: the umask is process-wide, so the test sets a restrictive one
+// and restores it.
+func TestUnpackInstallsFilesWithFixedModes(t *testing.T) {
+	old := syscall.Umask(0o077)
+	t.Cleanup(func() { syscall.Umask(old) })
+	b := testZip(t, map[string]string{"bootstrap": "#!/bin/sh\n", "lib/data.txt": "data"})
+	dir := t.TempDir()
+	require.NoError(t, unpack(bytes.NewReader(b), specFor(b), dir, devLimits))
+
+	pfi, err := os.Stat(filepath.Join(dir, "lib"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o755), pfi.Mode().Perm(), "implicit parent dir")
+
+	fi, err := os.Stat(filepath.Join(dir, "bootstrap"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o755), fi.Mode().Perm(), "executable")
+	fi, err = os.Stat(filepath.Join(dir, "lib", "data.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o644), fi.Mode().Perm())
+	got, err := os.ReadFile(filepath.Join(dir, "lib", "data.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, "data", string(got))
+}
+
+func TestUnpackRefusesBeforeWritingAnything(t *testing.T) {
+	symlink := &zip.FileHeader{Name: "link", Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: 1}
+	symlink.SetMode(fs.ModeSymlink | 0o777)
+	fifo := &zip.FileHeader{Name: "pipe", Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: 1}
+	fifo.SetMode(fs.ModeNamedPipe | 0o644)
+	// The second size would wrap a plain running sum back under the limit.
+	big := func(name string, n uint64) *zip.FileHeader {
+		return &zip.FileHeader{Name: name, Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: n}
+	}
+	cases := map[string]struct {
+		zip  []byte
+		want string
+	}{
+		"parent traversal": {testZip(t, map[string]string{"a": "ok", "../escape": "x"}), "entry name refused"},
+		"nested traversal": {testZip(t, map[string]string{"a": "ok", "lib/../../escape": "x"}), "entry name refused"},
+		"absolute path":    {testZip(t, map[string]string{"a": "ok", "/etc/escape": "x"}), "entry name refused"},
+		"backslash":        {testZip(t, map[string]string{"a": "ok", `..\escape`: "x"}), "entry name refused"},
+		"symlink":          {rawZip(t, big("a", 1), symlink), "entries are refused"},
+		"named pipe":       {rawZip(t, big("a", 1), fifo), "entries are refused"},
+		"declared size":    {rawZip(t, big("a", devMaxUnzipBytes+1)), "over the"},
+		"wrapping sizes":   {rawZip(t, big("a", devMaxUnzipBytes), big("b", math.MaxUint64)), "over the"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := t.TempDir()
+			err := unpack(bytes.NewReader(tc.zip), specFor(tc.zip), dir, devLimits)
+			assert.ErrorContains(t, err, tc.want)
+			left, rerr := os.ReadDir(dir)
+			require.NoError(t, rerr)
+			assert.Empty(t, left, "files written before the refusal")
+		})
+	}
+}
+
+func TestUnpackRefusesTooManyEntries(t *testing.T) {
+	b := testZip(t, map[string]string{"a": "1", "b": "2", "c": "3"})
+	err := unpack(bytes.NewReader(b), specFor(b), t.TempDir(), unpackLimits{entries: 2, bytes: devMaxUnzipBytes})
+	assert.ErrorContains(t, err, "limit 2")
+}
+
+// The header claims a size within the limit, but the data is longer: the
+// actual bytes are refused, whatever the header says.
+func TestUnpackRefusesBytesBeyondTheLimit(t *testing.T) {
+	data := bytes.Repeat([]byte("a"), 64)
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	h := &zip.FileHeader{Name: "a", Method: zip.Store, CRC32: crc32.ChecksumIEEE(data), CompressedSize64: uint64(len(data)), UncompressedSize64: 8}
+	w, err := zw.CreateRaw(h)
+	require.NoError(t, err)
+	_, err = w.Write(data)
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	b := buf.Bytes()
+
+	dir := t.TempDir()
+	err = unpack(bytes.NewReader(b), specFor(b), dir, unpackLimits{entries: 4, bytes: 16})
+	require.Error(t, err)
+	fi, serr := os.Stat(filepath.Join(dir, "a"))
+	if serr == nil {
+		assert.LessOrEqual(t, fi.Size(), int64(17))
+	}
+}
+
+func TestUnpackChecksSizeAndHash(t *testing.T) {
+	b := testZip(t, map[string]string{"bootstrap": "x"})
+	wrongHash := specFor(b)
+	wrongHash.sha256[0] ^= 1
+	assert.ErrorContains(t, unpack(bytes.NewReader(b), wrongHash, t.TempDir(), devLimits), "sha256 mismatch")
+
+	longer := specFor(b)
+	longer.size++
+	assert.ErrorContains(t, unpack(bytes.NewReader(b), longer, t.TempDir(), devLimits), "read package")
+}
+
+func TestReadPackageSpecBounds(t *testing.T) {
+	sum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	cases := map[string]struct {
+		size, hash string
+		want       string
+	}{
+		"valid":     {"10", sum, ""},
+		"zero":      {"0", sum, "outside"},
+		"too large": {"16777217", sum, "outside"},
+		"not hex":   {"10", "zz", "64 hex digits"},
+		"short":     {"10", sum[:62], "64 hex digits"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := t.TempDir()
+			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash} {
+				require.NoError(t, os.MkdirAll(filepath.Join(dir, n), 0o700))
+				require.NoError(t, os.WriteFile(filepath.Join(dir, n, "raw"), []byte(v), 0o600))
+			}
+			spec, err := readPackageSpec(dir)
+			if tc.want == "" {
+				require.NoError(t, err)
+				assert.Equal(t, int64(10), spec.size)
+				return
+			}
+			assert.ErrorContains(t, err, tc.want)
+		})
+	}
+}
+
+// A pre-existing symlink where a directory belongs is refused, not followed
+// or chmodded.
+func TestMkdirsRefusesSymlinkComponent(t *testing.T) {
+	dir := t.TempDir()
+	outside := t.TempDir()
+	require.NoError(t, os.Chmod(outside, 0o700))
+	require.NoError(t, os.Symlink(outside, filepath.Join(dir, "lib")))
+	root, err := os.OpenRoot(dir)
+	require.NoError(t, err)
+	defer root.Close()
+	assert.ErrorContains(t, mkdirs(root, "lib/sub"), "not a directory")
+	fi, err := os.Stat(outside)
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o700), fi.Mode().Perm(), "symlink target was chmodded")
+}

cmd/lambda-agent/start_tcp.go

@@ -0,0 +1,56 @@
+//go:build !lambdatestlink
+
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+)
+
+// errNoProtocol is returned when the host sends anything: this build defines no
+// message protocol, so it accepts no work.
+var errNoProtocol = errors.New("host sent data, but this build accepts no invocations")
+
+func start(ctx context.Context) int {
+	code, err := defaultStartup().run(ctx)
+	if err != nil {
+		slog.Error("lambda-agent stopping", "exit", code, "err", err)
+	}
+	return code
+}
+
+// run authenticates to the host and holds the connection until it closes.
+func (s startup) run(ctx context.Context) (int, error) {
+	conn, code, err := s.connect(ctx)
+	if err != nil {
+		return code, err
+	}
+	defer conn.Close()
+	slog.Info("control connection open; invocations are not enabled in this build", "host", conn.RemoteAddr().String())
+	if err := awaitHostClose(ctx, conn); err != nil {
+		return exitLink, err
+	}
+	return exitClosed, nil
+}
+
+// awaitHostClose reads until the host closes the connection. Under TLS 1.3 a
+// host that rejects the guest certificate is only seen here, on first read.
+func awaitHostClose(ctx context.Context, conn *tls.Conn) error {
+	stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
+	defer stop()
+	n, err := conn.Read(make([]byte, 1))
+	switch {
+	case n > 0:
+		return errNoProtocol
+	case errors.Is(err, io.EOF):
+		return nil
+	case ctx.Err() != nil:
+		return fmt.Errorf("control connection: %w", context.Cause(ctx))
+	default:
+		return fmt.Errorf("control connection: %w", err)
+	}
+}

cmd/lambda-agent/start_tcp_test.go

@@ -0,0 +1,171 @@
+//go:build !lambdatestlink
+
+package main
+
+import (
+	"context"
+	"errors"
+	"net"
+	"os"
+	"testing"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/guestconn"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type result struct {
+	code int
+	err  error
+}
+
+func runAsync(ctx context.Context, s startup) <-chan result {
+	out := make(chan result, 1)
+	go func() {
+		code, err := s.run(ctx)
+		out <- result{code, err}
+	}()
+	return out
+}
+
+func TestStartupAuthenticatesAfterSealingFwCfg(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	g := newGuest(t, h.entries())
+	done := runAsync(ctx, g.startup())
+
+	conn := h.guest(t)
+	select {
+	case <-g.sealedAt:
+	default:
+		t.Fatal("guest dialled before fw_cfg was sealed")
+	}
+	_, err := os.Stat(g.root)
+	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg still readable while connected")
+
+	require.NoError(t, conn.Close())
+	res := wait(t, done, "startup to return")
+	assert.Equal(t, exitClosed, res.code)
+	assert.NoError(t, res.err)
+	assert.Equal(t, 1, g.unloads)
+}
+
+func TestStartupRefusesHostData(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	done := runAsync(ctx, newGuest(t, h.entries()).startup())
+
+	conn := h.guest(t)
+	defer conn.Close()
+	_, err := conn.Write([]byte{0})
+	require.NoError(t, err)
+	res := wait(t, done, "startup to return")
+	assert.Equal(t, exitLink, res.code)
+	assert.ErrorIs(t, res.err, errNoProtocol)
+}
+
+// The host pins another generation's guest; the guest learns of the refusal
+// on its first read and fails with a link error, never a clean close.
+func TestStartupFailsWhenHostRefusesGuest(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	other, err := guestconn.NewGeneration(scope, time.Hour)
+	require.NoError(t, err)
+	entries := h.entries()
+	wrong := other.GuestEntries(h.ln.Addr().String())
+	entries[lambdav1.EntryGuestKey], entries[lambdav1.EntryGuestCert] = wrong[lambdav1.EntryGuestKey], wrong[lambdav1.EntryGuestCert]
+	g := newGuest(t, entries)
+
+	res := wait(t, runAsync(ctx, g.startup()), "startup to return")
+	assert.Equal(t, exitLink, res.code)
+	require.Error(t, res.err)
+	assert.Contains(t, res.err.Error(), "bad certificate")
+	assert.Equal(t, 1, g.unloads)
+	select {
+	case a := <-h.conn:
+		require.Error(t, a.err, "host accepted a guest it does not pin")
+	default:
+	}
+}
+
+func TestStartupRefusesWithoutDialling(t *testing.T) {
+	h := newHost(t)
+	cases := map[string]struct {
+		entries map[string][]byte
+		unload  func(g *guest) func() error
+		want    string
+	}{
+		"missing entry": {
+			entries: func() map[string][]byte { e := h.entries(); delete(e, lambdav1.EntryHostCert); return e }(),
+			want:    "read host-cert",
+		},
+		"fw_cfg stays readable": {
+			entries: h.entries(),
+			unload:  func(*guest) func() error { return func() error { return errors.New("module is built in") } },
+			want:    "fw_cfg still readable",
+		},
+		"unload claims success but tree remains": {
+			entries: h.entries(),
+			unload:  func(*guest) func() error { return func() error { return nil } },
+			want:    "unload reported success",
+		},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			g := newGuest(t, tc.entries)
+			s := g.startup()
+			if tc.unload != nil {
+				s.unload = tc.unload(g)
+			}
+			code, err := s.run(context.Background())
+			assert.Equal(t, exitRefused, code)
+			assert.ErrorContains(t, err, tc.want)
+		})
+	}
+
+	// Nothing dialled the host in any case above.
+	require.NoError(t, h.ln.SetDeadline(time.Now().Add(50*time.Millisecond)))
+	_, err := h.ln.Accept()
+	var ne net.Error
+	require.ErrorAs(t, err, &ne)
+	assert.True(t, ne.Timeout())
+}
+
+// A missing entry still seals fw_cfg: the key may be present even when
+// another entry is not.
+func TestStartupSealsEvenWhenIdentityIsIncomplete(t *testing.T) {
+	h := newHost(t)
+	e := h.entries()
+	delete(e, lambdav1.EntryHostAddr)
+	g := newGuest(t, e)
+	code, err := g.startup().run(context.Background())
+	assert.Equal(t, exitRefused, code)
+	assert.ErrorContains(t, err, "read host-addr")
+	assert.Equal(t, 1, g.unloads)
+	_, statErr := os.Stat(g.root)
+	assert.ErrorIs(t, statErr, os.ErrNotExist)
+}
+
+func TestStartupStopsWithContext(t *testing.T) {
+	ctx, cancel := context.WithCancelCause(context.Background())
+	defer cancel(nil)
+	h := newHost(t)
+	h.accept(ctx)
+	done := runAsync(ctx, newGuest(t, h.entries()).startup())
+	conn := h.guest(t)
+	defer conn.Close()
+
+	reason := errors.New("guest shutting down")
+	cancel(reason)
+	res := wait(t, done, "startup to return")
+	assert.Equal(t, exitLink, res.code)
+	assert.ErrorIs(t, res.err, reason)
+}

cmd/lambda-agent/start_testlink.go

@@ -0,0 +1,138 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+)
+
+const (
+	runtimeAddr = "127.0.0.1:9001"
+	taskRoot    = "/var/task"
+)
+
+var handler = flag.String("handler", os.Getenv("_HANDLER"), "function handler passed to the bootstrap")
+
+// testlink relays one invocation over the test carrier on the authenticated
+// connection; the carrier is not the guest protocol.
+type testlink struct {
+	startup
+	pkg            taskPackage
+	runtimeAddr    string
+	startBootstrap func(ctx context.Context, env []string) (*exec.Cmd, error)
+}
+
+func start(ctx context.Context) int {
+	t := testlink{
+		startup:     defaultStartup(),
+		pkg:         devTaskPackage(),
+		runtimeAddr: runtimeAddr,
+		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
+			return agent.StartBootstrap(ctx, taskRoot+"/bootstrap", env, os.Stdout, os.Stderr)
+		},
+	}
+	code, err := t.run(ctx)
+	if err != nil {
+		slog.Error("lambda-agent stopping", "exit", code, "err", err)
+	}
+	return code
+}
+
+func (t testlink) run(parent context.Context) (int, error) {
+	// The package values are fw_cfg entries, so read them before connect seals it.
+	spec, specErr := readPackageSpec(t.fwCfgDir)
+	conn, code, err := t.connect(parent)
+	if err != nil {
+		return code, errors.Join(err, specErr)
+	}
+	defer conn.Close()
+	if specErr != nil {
+		return exitRefused, specErr
+	}
+	link := hostlink.NewTestStream(conn)
+
+	// The handshake can return before the host refuses the guest, so nothing
+	// tenant-facing starts until the host's first valid frame arrives.
+	first, err := link.Next(parent)
+	if errors.Is(err, hostlink.ErrShutdown) {
+		return exitClosed, nil
+	}
+	if err != nil {
+		return exitLink, fmt.Errorf("first frame from host: %w", err)
+	}
+	if parent.Err() != nil {
+		return exitLink, context.Cause(parent)
+	}
+	if err := t.pkg.install(spec); err != nil {
+		return exitRefused, fmt.Errorf("install package: %w", err)
+	}
+
+	srv := runtimeapi.New()
+	ln, err := net.Listen("tcp", t.runtimeAddr)
+	if err != nil {
+		return exitLink, fmt.Errorf("listen runtime api: %w", err)
+	}
+	hs := &http.Server{Handler: srv.Handler(), ReadHeaderTimeout: 10 * time.Second}
+	go func() { _ = hs.Serve(ln) }()
+	defer hs.Close()
+
+	ctx, cancel := context.WithCancelCause(parent)
+	defer cancel(nil)
+	cmd, err := t.startBootstrap(ctx, []string{
+		"AWS_LAMBDA_RUNTIME_API=" + ln.Addr().String(),
+		"_HANDLER=" + *handler,
+		"LAMBDA_TASK_ROOT=" + taskRoot,
+		"PATH=/bin:/sbin:/usr/bin",
+	})
+	if err != nil {
+		return exitLink, fmt.Errorf("start bootstrap: %w", err)
+	}
+	exited := make(chan struct{})
+	go func() {
+		defer close(exited)
+		err := cmd.Wait()
+		cancel(fmt.Errorf("bootstrap exited: %w", err))
+	}()
+
+	err = agent.Run(ctx, &oneInvocation{Link: link, first: &first}, srv)
+	cancel(errors.New("agent stopping"))
+	<-exited
+	if err != nil {
+		return exitLink, err
+	}
+	return exitClosed, nil
+}
+
+// oneInvocation serves the host's first request and then stops the agent:
+// each test VM runs a single invocation.
+type oneInvocation struct {
+	hostlink.Link
+	first *hostlink.Request
+}
+
+var _ hostlink.Link = (*oneInvocation)(nil)
+
+func (o *oneInvocation) Next(ctx context.Context) (hostlink.Request, error) {
+	if err := ctx.Err(); err != nil {
+		return hostlink.Request{}, err
+	}
+	if o.first == nil {
+		return hostlink.Request{}, hostlink.ErrShutdown
+	}
+	req := *o.first
+	o.first = nil
+	return req, nil
+}

cmd/lambda-agent/start_testlink_test.go

@@ -0,0 +1,246 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/guestconn"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const helperEnv = "LAMBDA_AGENT_TEST_BOOTSTRAP"
+
+// TestHelperBootstrap is the fake function bootstrap, run as a subprocess: it
+// echoes one invocation through the Runtime API, then waits for the next.
+func TestHelperBootstrap(t *testing.T) {
+	if os.Getenv(helperEnv) != "1" {
+		return
+	}
+	base := "http://" + os.Getenv("AWS_LAMBDA_RUNTIME_API") + "/2018-06-01/runtime/invocation/"
+	resp, err := http.Get(base + "next")
+	require.NoError(t, err)
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	resp.Body.Close()
+	id := resp.Header.Get("Lambda-Runtime-Aws-Request-Id")
+	post, err := http.Post(base+id+"/response", "application/json", bytes.NewReader(body))
+	require.NoError(t, err)
+	post.Body.Close()
+	_, _ = http.Get(base + "next")
+}
+
+type bootstrapStarts struct {
+	n       int
+	started chan struct{}
+	task    string
+	sealed  bool
+}
+
+// testlink builds a guest whose fw_cfg also carries a controlled package,
+// installed into a temp dir without mounting anything.
+func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) testlink {
+	t.Helper()
+	zipBytes := testZip(t, map[string]string{"bootstrap": "#!/bin/sh\n", "lib/data.txt": "data"})
+	disk := filepath.Join(t.TempDir(), "disk")
+	require.NoError(t, os.WriteFile(disk, zipBytes, 0o600))
+	sum := sha256.Sum256(zipBytes)
+	entries[entryPackageSize] = []byte(strconv.Itoa(len(zipBytes)))
+	entries[entryPackageSHA256] = []byte(hex.EncodeToString(sum[:]))
+	g := newGuest(t, entries)
+	b.started = make(chan struct{}, 1)
+	b.task = t.TempDir()
+	return testlink{
+		startup: g.startup(),
+		pkg: taskPackage{
+			device: disk, dir: b.task,
+			loadDriver: func() error { return nil },
+			mountDir:   func(string) error { return nil },
+			sealDir:    func(string) error { b.sealed = true; return nil },
+		},
+		runtimeAddr: "127.0.0.1:0",
+		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
+			b.n++
+			b.started <- struct{}{}
+			cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestHelperBootstrap$")
+			cmd.Env = append(env, helperEnv+"=1")
+			return cmd, cmd.Start()
+		},
+	}
+}
+
+func runTestlinkAsync(ctx context.Context, tl testlink) <-chan result {
+	out := make(chan result, 1)
+	go func() {
+		code, err := tl.run(ctx)
+		out <- result{code, err}
+	}()
+	return out
+}
+
+type result struct {
+	code int
+	err  error
+}
+
+func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
+
+	conn := h.guest(t)
+	defer conn.Close()
+	select {
+	case <-b.started:
+		t.Fatal("bootstrap started before the host sent a frame")
+	case <-time.After(100 * time.Millisecond):
+	}
+	hs := hostlink.NewTestStream(conn)
+	req := hostlink.Request{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{"a":40,"b":2}`)}
+	require.NoError(t, hs.HostSend(req))
+	got, err := hs.HostResult(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, hostlink.Response{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", Payload: req.Payload}, got)
+
+	// A second invocation on the same VM is never served. The agent may
+	// already have closed the connection, so a failed send is a refusal too.
+	second := req
+	second.RequestID = "r-2"
+	sendErr := hs.HostSend(second)
+	res := wait(t, done, "agent to stop after one invocation")
+	assert.Equal(t, exitClosed, res.code)
+	assert.NoError(t, res.err)
+	if sendErr == nil {
+		_, err = hs.HostResult(ctx)
+		assert.Error(t, err, "agent answered a second invocation")
+	}
+	assert.Equal(t, 1, b.n)
+	assert.True(t, b.sealed, "task dir not made read-only")
+	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
+}
+
+func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
+
+	conn := h.guest(t)
+	defer conn.Close()
+	require.NoError(t, hostlink.NewTestStream(conn).HostShutdown())
+	res := wait(t, done, "agent to stop")
+	assert.Equal(t, exitClosed, res.code)
+	assert.NoError(t, res.err)
+	assert.Zero(t, b.n)
+}
+
+// The guest's handshake completes, but the host pins another generation's
+// guest; the refusal arrives on first read and the bootstrap never starts.
+func TestTestlinkHostRefusalNeverStartsBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	other, err := guestconn.NewGeneration(scope, time.Hour)
+	require.NoError(t, err)
+	entries := h.entries()
+	wrong := other.GuestEntries(h.ln.Addr().String())
+	entries[lambdav1.EntryGuestKey], entries[lambdav1.EntryGuestCert] = wrong[lambdav1.EntryGuestKey], wrong[lambdav1.EntryGuestCert]
+	var b bootstrapStarts
+
+	res := wait(t, runTestlinkAsync(ctx, b.testlink(t, entries)), "agent to stop")
+	assert.Equal(t, exitLink, res.code)
+	require.Error(t, res.err)
+	assert.Contains(t, res.err.Error(), "bad certificate")
+	assert.Zero(t, b.n)
+}
+
+func TestTestlinkHostCloseBeforeFrameNeverStartsBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
+
+	require.NoError(t, h.guest(t).Close())
+	res := wait(t, done, "agent to stop")
+	assert.Equal(t, exitLink, res.code)
+	assert.ErrorContains(t, res.err, "first frame from host")
+	assert.Zero(t, b.n)
+}
+
+// A cancelled context is reported before the buffered request is delivered,
+// and the request stays undelivered.
+func TestOneInvocationHonoursCancelledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	o := &oneInvocation{first: &hostlink.Request{RequestID: "r-1"}}
+	_, err := o.Next(ctx)
+	require.ErrorIs(t, err, context.Canceled)
+
+	req, err := o.Next(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, "r-1", req.RequestID)
+}
+
+// A package that fails its check is refused after authentication, and the
+// bootstrap never starts.
+func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	tl := b.testlink(t, h.entries())
+	tl.pkg.device = filepath.Join(t.TempDir(), "other")
+	require.NoError(t, os.WriteFile(tl.pkg.device, testZip(t, map[string]string{"bootstrap": "x"}), 0o600))
+	done := runTestlinkAsync(ctx, tl)
+
+	conn := h.guest(t)
+	defer conn.Close()
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{RequestID: "r-1"}))
+	res := wait(t, done, "agent to stop")
+	assert.Equal(t, exitRefused, res.code)
+	assert.ErrorContains(t, res.err, "install package")
+	assert.Zero(t, b.n)
+	assert.False(t, b.sealed)
+}
+
+// Missing package values still seal fw_cfg, and the agent refuses before
+// waiting for any frame.
+func TestTestlinkMissingPackageSpecRefuses(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	tl := b.testlink(t, h.entries())
+	require.NoError(t, os.RemoveAll(filepath.Join(tl.fwCfgDir, entryPackageSHA256)))
+
+	res := wait(t, runTestlinkAsync(ctx, tl), "agent to stop")
+	assert.Equal(t, exitRefused, res.code)
+	assert.ErrorContains(t, res.err, "read "+entryPackageSHA256)
+	assert.Zero(t, b.n)
+	_, err := os.Stat(tl.fwCfgRoot)
+	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg not sealed")
+}

contracts/cluster/v1/facts.go

@@ -0,0 +1,217 @@
+package clusterv1
+
+import "regexp"
+
+const (
+	// NodeFactsSubject fans out a request for each daemon's hardware and
+	// cabling description. Its reply payload is NodeFacts.
+	NodeFactsSubject = "spinifex.node.facts"
+
+	nodeNetTestPrefix = "spinifex.node.nettest."
+)
+
+var nodeIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
+
+// ValidNodeID reports whether a node name is safe to place in a subject: no
+// NATS token separator or wildcard can get through.
+func ValidNodeID(node string) bool {
+	return nodeIDPattern.MatchString(node)
+}
+
+// NodeFactsOfSubject asks one node alone for its NodeFacts.
+func NodeFactsOfSubject(node string) string {
+	return NodeFactsSubject + "." + node
+}
+
+// NetTestServeSubject asks one node to start a one-shot throughput server.
+func NetTestServeSubject(node string) string {
+	return nodeNetTestPrefix + node + ".serve"
+}
+
+// NetTestRunSubject asks one node to run the client side of a throughput test.
+func NetTestRunSubject(node string) string {
+	return nodeNetTestPrefix + node + ".run"
+}
+
+// Planes a node's NICs can carry, named as the installer names them.
+const (
+	PlaneWAN = "wan"
+	PlaneLAN = "lan"
+	PlaneVPC = "vpc"
+)
+
+// NetTestPort is the TCP port reserved for node-to-node throughput tests.
+const NetTestPort = 5001
+
+// NodeFacts describes one node's hardware, OS and cabling. Fields a host
+// cannot report are left empty, and Sources says why.
+type NodeFacts struct {
+	Node        string            `json:"node"`
+	CollectedAt string            `json:"collected_at,omitempty"`
+	FormFactor  string            `json:"form_factor"`
+	Virt        string            `json:"virt,omitempty"`
+	System      SystemFacts       `json:"system"`
+	OS          OSFacts           `json:"os"`
+	CPU         CPUFacts          `json:"cpu"`
+	Memory      MemoryFacts       `json:"memory"`
+	NICs        []NICFacts        `json:"nics"`
+	Planes      []PlaneFacts      `json:"planes"`
+	Sources     map[string]string `json:"sources"`
+}
+
+// SystemFacts is the machine's identity from DMI.
+type SystemFacts struct {
+	Vendor       string `json:"vendor,omitempty"`
+	Product      string `json:"product,omitempty"`
+	Family       string `json:"family,omitempty"`
+	Version      string `json:"version,omitempty"`
+	Serial       string `json:"serial,omitempty"`
+	UUID         string `json:"uuid,omitempty"`
+	ChassisType  string `json:"chassis_type,omitempty"`
+	ChassisAsset string `json:"chassis_asset,omitempty"`
+	BoardVendor  string `json:"board_vendor,omitempty"`
+	BoardProduct string `json:"board_product,omitempty"`
+	BoardSerial  string `json:"board_serial,omitempty"`
+	BIOSVendor   string `json:"bios_vendor,omitempty"`
+	BIOSVersion  string `json:"bios_version,omitempty"`
+	BIOSDate     string `json:"bios_date,omitempty"`
+	DeviceModel  string `json:"device_model,omitempty"`
+}
+
+// OSFacts names the running OS and Spinifex build.
+type OSFacts struct {
+	Name          string `json:"name,omitempty"`
+	Kernel        string `json:"kernel,omitempty"`
+	Arch          string `json:"arch,omitempty"`
+	Version       string `json:"spx_version,omitempty"`
+	Commit        string `json:"spx_commit,omitempty"`
+	UptimeSeconds int64  `json:"uptime_seconds,omitempty"`
+	Hostname      string `json:"hostname,omitempty"`
+}
+
+// CPUFacts summarises the sockets. Threads and Cores are totals. Features are
+// the flags worth naming; Flags is every flag the CPU reports.
+type CPUFacts struct {
+	Model          string   `json:"model,omitempty"`
+	Vendor         string   `json:"vendor,omitempty"`
+	Sockets        int      `json:"sockets"`
+	CoresPerSocket int      `json:"cores_per_socket,omitempty"`
+	Cores          int      `json:"cores"`
+	Threads        int      `json:"threads"`
+	MaxMHz         int      `json:"max_mhz,omitempty"`
+	CurrentMHz     int      `json:"current_mhz,omitempty"`
+	Virtualization string   `json:"virtualization,omitempty"`
+	Features       []string `json:"features,omitempty"`
+	Flags          []string `json:"flags,omitempty"`
+}
+
+// MemoryFacts groups identical DIMMs so a 32-slot board reads as one line.
+type MemoryFacts struct {
+	TotalBytes    uint64        `json:"total_bytes"`
+	SlotsUsed     int           `json:"slots_used"`
+	SlotsTotal    int           `json:"slots_total"`
+	MaxCapacity   string        `json:"max_capacity,omitempty"`
+	ECC           string        `json:"ecc,omitempty"`
+	Groups        []MemoryGroup `json:"groups"`
+	OSTotalBytes  uint64        `json:"os_total_bytes,omitempty"`
+	OSAvailBytes  uint64        `json:"os_available_bytes,omitempty"`
+	HugePageBytes uint64        `json:"huge_page_bytes,omitempty"`
+}
+
+// MemoryGroup is a set of DIMMs alike in every field shown. ConfiguredMTs
+// below SpeedMTs means the board runs them slower than rated.
+type MemoryGroup struct {
+	Count         int      `json:"count"`
+	SizeBytes     uint64   `json:"size_bytes"`
+	Type          string   `json:"type,omitempty"`
+	TypeDetail    string   `json:"type_detail,omitempty"`
+	FormFactor    string   `json:"form_factor,omitempty"`
+	SpeedMTs      int      `json:"speed_mts,omitempty"`
+	ConfiguredMTs int      `json:"configured_mts,omitempty"`
+	Manufacturer  string   `json:"manufacturer,omitempty"`
+	PartNumber    string   `json:"part_number,omitempty"`
+	Rank          string   `json:"rank,omitempty"`
+	Serials       []string `json:"serials,omitempty"`
+}
+
+// NICFacts is one physical network interface and, where LLDP reports it, the
+// switch port it is cabled to.
+type NICFacts struct {
+	Name      string        `json:"name"`
+	MAC       string        `json:"mac,omitempty"`
+	Driver    string        `json:"driver,omitempty"`
+	Firmware  string        `json:"firmware,omitempty"`
+	BusInfo   string        `json:"bus_info,omitempty"`
+	Model     string        `json:"model,omitempty"`
+	SpeedMbps int           `json:"speed_mbps,omitempty"`
+	MaxMbps   int           `json:"max_mbps,omitempty"`
+	Duplex    string        `json:"duplex,omitempty"`
+	Port      string        `json:"port,omitempty"`
+	MTU       int           `json:"mtu,omitempty"`
+	State     string        `json:"state"`
+	Master    string        `json:"master,omitempty"`
+	Planes    []string      `json:"planes,omitempty"`
+	Neighbor  *LLDPNeighbor `json:"neighbor,omitempty"`
+}
+
+// LLDPNeighbor is what the switch on the other end of a NIC advertises.
+type LLDPNeighbor struct {
+	ChassisID         string   `json:"chassis_id"`
+	SystemName        string   `json:"system_name,omitempty"`
+	SystemDescription string   `json:"system_description,omitempty"`
+	PortID            string   `json:"port_id,omitempty"`
+	PortDescription   string   `json:"port_description,omitempty"`
+	Capabilities      []string `json:"capabilities,omitempty"`
+}
+
+// PlaneFacts is the address a plane uses on this node and the NICs under it.
+type PlaneFacts struct {
+	Plane   string   `json:"plane"`
+	Bridge  string   `json:"bridge,omitempty"`
+	Address string   `json:"address,omitempty"`
+	Prefix  int      `json:"prefix,omitempty"`
+	NICs    []string `json:"nics,omitempty"`
+	MTU     int      `json:"mtu,omitempty"`
+}
+
+// NetTestServeRequest starts a server bound to the node's address on Plane.
+type NetTestServeRequest struct {
+	Plane           string `json:"plane"`
+	DurationSeconds int    `json:"duration_seconds"`
+}
+
+// NetTestServeResponse names the address the server listens on.
+type NetTestServeResponse struct {
+	Address string `json:"address,omitempty"`
+	Port    int    `json:"port,omitempty"`
+	Error   string `json:"error,omitempty"`
+}
+
+// NetTestRunRequest runs the client towards Address from the node's own
+// address on Plane. Address must be what Target reports for that plane.
+type NetTestRunRequest struct {
+	Plane           string `json:"plane"`
+	Target          string `json:"target"`
+	Address         string `json:"address"`
+	Port            int    `json:"port"`
+	DurationSeconds int    `json:"duration_seconds"`
+	Streams         int    `json:"streams"`
+}
+
+// NetTestRunResponse is one direction of a pair's measurement.
+type NetTestRunResponse struct {
+	SourceAddress string    `json:"source_address,omitempty"`
+	BitsPerSecond float64   `json:"bits_per_second"`
+	Bytes         int64     `json:"bytes"`
+	Seconds       float64   `json:"seconds"`
+	Streams       int       `json:"streams"`
+	StreamBPS     []float64 `json:"stream_bps,omitempty"`
+	RTTMinMillis  float64   `json:"rtt_min_ms,omitempty"`
+	RTTAvgMillis  float64   `json:"rtt_avg_ms,omitempty"`
+	RTTMaxMillis  float64   `json:"rtt_max_ms,omitempty"`
+	PacketLoss    float64   `json:"packet_loss_percent,omitempty"`
+	LinkMbps      int       `json:"link_mbps,omitempty"`
+	MTU           int       `json:"mtu,omitempty"`
+	Tool          string    `json:"tool,omitempty"`
+	Error         string    `json:"error,omitempty"`
+}

contracts/lambda/v1/README.md

@@ -0,0 +1,66 @@
+# Lambda v1 contracts
+
+This package will own the Lambda process contracts declared by ADR-0009:S13.
+It holds the L-GUEST declaration and, in `guest.go`, the connection vocabulary both ends share: the scope, the URI SAN roles, the ALPN identifier and the fw_cfg entry names. The transport and authentication are decided (ADR-0009:S17); the invocation message types and framing are not chosen, so no message types are defined here.
+It is not an AWS tenant-facing API.
+
+## L-GUEST: node executor to guest agent
+
+**Owner:** the Lambda domain.
+
+**Producers and consumers:** the node-local executor (`runtime/compute/lambda`) and the in-guest Lambda agent (`agents/lambda`), in both directions.
+
+**Operations:** bootstrap with scoped material, readiness, invoke, completion, bounded logs, shutdown.
+
+**Transport:** TCP over the microVM's network (ADR-0009:S17).
+The agent dials one connection to a listener the executor owns for the environment, and the guest exposes no listener for host control.
+At most one authenticated control connection is active per environment, and the agent may reconnect; how a prior connection and its in-flight work are handled is open (ADR-0009 M1).
+The network is shared and untrusted: no source address, port or bridge membership is identity or authority.
+Tenant code may be able to reach the listener, but it cannot open the agent's connection or key material, so any connection it makes fails authentication.
+
+**Authentication:** mutual TLS 1.3, bound to the environment ID and realization generation, using material the user runtime cannot read (ADR-0009:S17).
+For each generation the executor creates ephemeral self-signed host and guest certificates whose single URI SAN is `spinifex-lambda://<Region>/<account>/<function ID>/<environment ID>/<generation>/<role>`, with role `host` or `guest`.
+Each end validates the peer certificate against the one pinned peer certificate and separately requires that URI SAN with the opposite role; hostname verification is not relied on.
+ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake.
+Under TLS 1.3 the guest's handshake can complete before the host refuses its certificate, so the guest treats the connection as accepted only after the host's first message.
+
+**Bootstrap material:** the executor passes these fw_cfg entries under `opt/spinifex/lambda/`, and the agent reads them as root and unloads `qemu_fw_cfg` before user code starts.
+
+| Entry | Content |
+|---|---|
+| `guest-key` | Guest private key, PEM |
+| `guest-cert` | Guest certificate, PEM |
+| `host-cert` | The one pinned host certificate, PEM |
+| `expect` | `<Region>/<account>/<function ID>/<environment ID>/<generation>` |
+| `host-addr` | Host listener as `host:port` |
+
+The guest's own interface settings come from `opt/spinifex/netcfg`, the entry other system VMs use.
+The agent unloads `qemu_fw_cfg` even when an entry is missing, and refuses to start if the fw_cfg tree is still readable afterwards.
+Until the message format exists, the agent authenticates and holds the connection but accepts no host message and starts no user code.
+
+**Authority:** one environment's attempt only.
+The user runtime never holds these keys, and sharing the guest with the agent gives user code no platform authority.
+
+**Idempotency and fencing:** an invoke carries the attempt-grant identity and the attempt ID.
+The agent accepts a run command only under a current grant, and a completion under a superseded grant is discarded and reported.
+How a grant is enforced and proven ended is open (ADR-0009 M1).
+
+**Errors and versions:** the agent and executor fail closed on an unknown protocol version or an identity mismatch.
+
+**Readiness:** environment readiness needs authenticated, current-generation agent evidence; a running hypervisor process does not count.
+
+**Payloads:** the contract must carry a synchronous payload and response of up to 6 MB and an asynchronous payload of up to 1 MB, plus framing, over a path whose size bound is declared.
+How payloads move is open (ADR-0009 M3).
+
+### Compatibility boundary
+
+The agent ships inside the runtime image pinned by each published version, so it is outside the coordinated release train.
+The executor supports every L-GUEST major carried by a still-supported runtime image.
+Retiring an image follows a published runtime-deprecation lifecycle that blocks create and update before it blocks invoke.
+
+Evidence required before the first agent image ships: a function on an old image invoked by a new executor, and the reverse during rollback.
+
+## L-RUNTIME
+
+The guest agent serves the AWS Lambda Runtime API, version `2018-06-01`, to the user runtime at `AWS_LAMBDA_RUNTIME_API`.
+Spinifex does not version this AWS contract, and conformance is measured against AWS.

contracts/lambda/v1/guest.go

@@ -0,0 +1,54 @@
+// Package lambdav1 holds the Lambda process-contract vocabulary shared by the
+// node executor and the guest agent. See README.md for the declarations.
+package lambdav1
+
+import (
+	"fmt"
+	"strings"
+)
+
+// ALPNv1 names L-GUEST major version 1; a peer offering any other fails the handshake.
+const ALPNv1 = "spinifex-lambda-guest/1"
+
+// FwCfgPrefix prefixes every fw_cfg entry the executor passes the guest.
+const FwCfgPrefix = "opt/spinifex/lambda/"
+
+// fw_cfg entry names under FwCfgPrefix.
+const (
+	EntryGuestKey  = "guest-key"
+	EntryGuestCert = "guest-cert"
+	EntryHostCert  = "host-cert"
+	EntryScope     = "expect"
+	EntryHostAddr  = "host-addr"
+)
+
+// Roles named as the last URI SAN path segment.
+const (
+	RoleHost  = "host"
+	RoleGuest = "guest"
+)
+
+const uriScheme = "spinifex-lambda"
+
+// Scope names one environment generation:
+// Region/account/function ID/environment ID/generation.
+type Scope string
+
+// ParseScope accepts exactly five non-empty, slash-separated segments.
+func ParseScope(s string) (Scope, error) {
+	parts := strings.Split(s, "/")
+	if len(parts) != 5 {
+		return "", fmt.Errorf("lambdav1: scope %q: want 5 segments, got %d", s, len(parts))
+	}
+	for _, p := range parts {
+		if p == "" || strings.ContainsAny(p, " \t\r\n?#%") {
+			return "", fmt.Errorf("lambdav1: scope %q: empty or invalid segment", s)
+		}
+	}
+	return Scope(s), nil
+}
+
+// URI is the single URI SAN a certificate for role in this scope carries.
+func (s Scope) URI(role string) string {
+	return uriScheme + "://" + string(s) + "/" + role
+}

contracts/lambda/v1/guest_test.go

@@ -0,0 +1,20 @@
+package lambdav1
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseScope(t *testing.T) {
+	s, err := ParseScope("ap-southeast-2/000000000001/fn-1/env-1/gen-1")
+	require.NoError(t, err)
+	assert.Equal(t, "spinifex-lambda://ap-southeast-2/000000000001/fn-1/env-1/gen-1/host", s.URI(RoleHost))
+	for _, bad := range []string{
+		"", "a/b/c/d", "a/b/c/d/e/f", "a//c/d/e", "a/b/c/d/e/", "a/b/c/d/e?x", "a/b/c/d/e#x", "a b/b/c/d/e",
+	} {
+		_, err := ParseScope(bad)
+		assert.Error(t, err, bad)
+	}
+}

contracts/operator/v1/admin.go

@@ -25,6 +25,10 @@ const (
 	DeletionStateRunning   = "RUNNING"
 	DeletionStateCompleted = "COMPLETED"
 	DeletionStateFailed    = "FAILED"
+
+	// DeletionStateExercised ends a non-finalizing run: the account is left
+	// TERMINATING and a later DeleteAccount takes the same operation over.
+	DeletionStateExercised = "EXERCISED"
 )
 
 // CreateAccountRequest is the /admin/CreateAccount request body.
@@ -52,6 +56,10 @@ type DeleteAccountRequest struct {
 	ClientToken string `json:"clientToken"`
 	Force       bool   `json:"force,omitempty"`
 	DryRun      bool   `json:"dryRun,omitempty"`
+
+	// NonFinalizing quiesces the account, cleans the quiesced EC2 platform
+	// kinds and stops. No legacy stage runs and the account record stays.
+	NonFinalizing bool `json:"nonFinalizing,omitempty"`
 }
 
 // DescribeAccountDeletionRequest is the /admin/DescribeAccountDeletion body.
@@ -74,3 +82,129 @@ type ListAccountsResponse struct {
 	Accounts []AccountSummary `json:"accounts"`
 	Count    int              `json:"count"`
 }
+
+// IAMQuiescence is the coordinator's recorded view of IAM quiescence for one
+// fence. Recording only: it admits nothing. QuiescedScope is filled only for a
+// non-finalizing run, and holds as of EvaluatedAt: a write after that is not caught.
+type IAMQuiescence struct {
+	OperationID   string            `json:"operationId"`
+	Fence         uint64            `json:"fence"`
+	Generation    uint64            `json:"generation"`
+	QuiescedScope []string          `json:"quiescedScope"`
+	Unresolved    []UnresolvedScope `json:"unresolved"`
+	Consumers     []ConsumerBasis   `json:"consumers"`
+	Writers       []AccountedWriter `json:"writers"`
+	Inventory     []InventoryStatus `json:"inventory"`
+	EvaluatedAt   string            `json:"evaluatedAt"`
+}
+
+// UnresolvedScope names a scope or writer that blocks quiescence, with a fixed
+// reason. It is never a timeout.
+type UnresolvedScope struct {
+	Scope  string `json:"scope"`
+	Class  string `json:"class"`
+	Reason string `json:"reason"`
+}
+
+// ConsumerBasis cites an enforcement consumer and the accepted contract that
+// lets it count as having applied the generation.
+type ConsumerBasis struct {
+	Consumer string `json:"consumer"`
+	Basis    string `json:"basis"`
+}
+
+// AccountedWriter is a writer that cannot be fenced and is covered by the
+// owner's inventory instead. Ref names the tracked work that closes the gap.
+type AccountedWriter struct {
+	Writer string `json:"writer"`
+	Basis  string `json:"basis"`
+	Ref    string `json:"ref"`
+}
+
+// InventoryStatus is the owner inventory's status for one kind at evaluation.
+type InventoryStatus struct {
+	Kind   string `json:"kind"`
+	Status string `json:"status"`
+}
+
+// Classes of an UnresolvedScope.
+const (
+	UnresolvedClassFence    = "fence"
+	UnresolvedClassConsumer = "consumer"
+	UnresolvedClassWriter   = "writer"
+)
+
+// Fixed reasons of an UnresolvedScope.
+const (
+	ReasonFenceMismatch                = "fence-mismatch"
+	ReasonIAMReadbackFailed            = "iam-readback-failed"
+	ReasonNoAcceptedEnforcement        = "no-accepted-enforcement-contract"
+	ReasonNoCompleteOwnerInventory     = "no-complete-owner-inventory"
+	ReasonParticipantNotDeclared       = "participant-not-declared"
+	ReasonOwnerInventoryIncomplete     = "owner-inventory-incomplete"
+	ReasonFinalInventoryRereadNotWired = "final-inventory-reread-not-wired"
+)
+
+// ExerciseResult is what a non-finalizing run did after IAM quiescence.
+type ExerciseResult struct {
+	// Skipped is the fixed reason no cleanup ran; Cleanup is then absent.
+	Skipped    string           `json:"skipped,omitempty"`
+	Cleanup    *PlatformCleanup `json:"cleanup,omitempty"`
+	Accounting []KindAccounting `json:"accounting,omitempty"`
+}
+
+// PlatformCleanup is the EC2 platform cleanup result. It carries resource IDs,
+// fixed reasons and counts only, never names, tags, key material or data.
+type PlatformCleanup struct {
+	Kinds  []CleanupKind  `json:"kinds"`
+	Items  []CleanupItem  `json:"items,omitempty"`
+	Counts []CleanupCount `json:"counts,omitempty"`
+}
+
+// CleanupKind is one kind's scan status after the cleanup's own re-scan.
+type CleanupKind struct {
+	Kind              string   `json:"kind"`
+	Status            string   `json:"status"`
+	IncompleteReasons []string `json:"incompleteReasons,omitempty"`
+}
+
+// CleanupItem is one resource by its ID with what the cleanup did to it.
+type CleanupItem struct {
+	ResourceID  string `json:"resourceId"`
+	Kind        string `json:"kind"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	MemberCount int    `json:"memberCount,omitempty"`
+}
+
+// CleanupCount is the number of records of one class that have no public ID.
+type CleanupCount struct {
+	Kind        string `json:"kind"`
+	Class       string `json:"class"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	Count       int    `json:"count"`
+}
+
+// KindAccounting is the final inventory re-read held against the cleanup result
+// for one kind. Quiescence is not rewritten from it.
+type KindAccounting struct {
+	Kind      string `json:"kind"`
+	Status    string `json:"status"`
+	Reason    string `json:"reason,omitempty"`
+	Remaining int    `json:"remaining"`
+}
+
+// Accounting statuses and fixed reasons of a KindAccounting.
+const (
+	AccountingAccounted   = "accounted"
+	AccountingUnaccounted = "unaccounted"
+
+	AccountingReasonRereadIncomplete = "reread-incomplete"
+	AccountingReasonUnnamedRemainder = "unnamed-remainder"
+)
+
+// Fixed reasons an exercise ran no cleanup.
+const (
+	ExerciseSkippedNoQuiescedKind = "no-quiesced-kind"
+)

contracts/operator/v1/admin_test.go

@@ -43,6 +43,27 @@ func TestAdminWire_DeletionStates(t *testing.T) {
 	require.Equal(t, "RUNNING", DeletionStateRunning)
 	require.Equal(t, "COMPLETED", DeletionStateCompleted)
 	require.Equal(t, "FAILED", DeletionStateFailed)
+	require.Equal(t, "EXERCISED", DeletionStateExercised)
+}
+
+func TestAdminWire_NonFinalizingDeleteRequest(t *testing.T) {
+	goldenJSON(t, DeleteAccountRequest{AccountID: "000000000042", AccountName: "acme", ClientToken: "tok-3", NonFinalizing: true},
+		`{"accountId":"000000000042","accountName":"acme","clientToken":"tok-3","nonFinalizing":true}`)
+}
+
+func TestAdminWire_ExerciseResult(t *testing.T) {
+	goldenJSON(t, ExerciseResult{
+		Cleanup: &PlatformCleanup{
+			Kinds:  []CleanupKind{{Kind: "key-pair", Status: "observed-clean-at-scan-time"}},
+			Items:  []CleanupItem{{ResourceID: "pg-1", Kind: "placement-group", Disposition: "blocked", Reason: "instance-members", MemberCount: 1}},
+			Counts: []CleanupCount{{Kind: "key-pair", Class: "public-key-object-without-metadata", Disposition: "deleted", Count: 2}},
+		},
+		Accounting: []KindAccounting{{Kind: "placement-group", Status: AccountingAccounted, Remaining: 1}},
+	}, `{"cleanup":{"kinds":[{"kind":"key-pair","status":"observed-clean-at-scan-time"}],
+		"items":[{"resourceId":"pg-1","kind":"placement-group","disposition":"blocked","reason":"instance-members","memberCount":1}],
+		"counts":[{"kind":"key-pair","class":"public-key-object-without-metadata","disposition":"deleted","count":2}]},
+		"accounting":[{"kind":"placement-group","status":"accounted","remaining":1}]}`)
+	goldenJSON(t, ExerciseResult{Skipped: ExerciseSkippedNoQuiescedKind}, `{"skipped":"no-quiesced-kind"}`)
 }
 
 func TestAdminWire_ListAccountsResponse(t *testing.T) {

contracts/telemetry/v1/node_live.go

@@ -0,0 +1,53 @@
+package telemetryv1
+
+import "regexp"
+
+// Node live subjects mirror the instance ones, keyed by node name. Only the
+// super-admin account's gateway handler subscribes to them.
+const (
+	nodeLiveWatchSubjectPrefix  = "telemetry.watch.node."
+	nodeLiveSampleSubjectPrefix = "telemetry.live.node."
+)
+
+var liveNodePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
+
+// ValidLiveNode reports whether a node name is safe to place in a subject.
+func ValidLiveNode(node string) bool {
+	return liveNodePattern.MatchString(node)
+}
+
+// NodeLiveWatchSubject is where a viewer asks a node to keep sampling itself.
+// A request on it is answered with a NodeLiveRing.
+func NodeLiveWatchSubject(node string) string {
+	return nodeLiveWatchSubjectPrefix + node
+}
+
+// NodeLiveSampleSubject carries one NodeLiveSample per period while watched.
+func NodeLiveSampleSubject(node string) string {
+	return nodeLiveSampleSubjectPrefix + node
+}
+
+// NodeLiveRing answers a watch with the samples newer than its SinceMillis.
+type NodeLiveRing struct {
+	Samples []NodeLiveSample `json:"samples"`
+}
+
+// NodeLiveSample is one host measurement. Rates are per second over the
+// interval ending at TimeMillis.
+type NodeLiveSample struct {
+	TimeMillis int64   `json:"t"`
+	CPUPercent float64 `json:"cpu"`
+	// MemoryUsedPercent is used memory less page cache, as free(1) reports it.
+	MemoryUsedPercent float64 `json:"mem"`
+	Load1             float64 `json:"load1"`
+	// Planes holds the traffic of each plane's physical NICs, keyed wan, lan, vpc.
+	Planes map[string]NodeLivePlane `json:"planes"`
+}
+
+// NodeLivePlane is one plane's traffic from the host's side.
+type NodeLivePlane struct {
+	NetworkIn  float64 `json:"netIn"`
+	NetworkOut float64 `json:"netOut"`
+	PacketsIn  float64 `json:"pktIn"`
+	PacketsOut float64 `json:"pktOut"`
+}

docs/DESIGN.md

@@ -65,9 +65,10 @@ The EC2 registration (`spinifex/domains/ec2/awsapi`, `NewRegistration`) parses t
 The gateway communicates with daemons via NATS request/response. Most calls go through `natsmsg.NATSRequest`, which marshals the input, attaches the account ID as a NATS header, and unmarshals the typed response:
 
 ```go
-func (s *NATSInstanceService) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
-    topic := fmt.Sprintf("ec2.RunInstances.%s", aws.StringValue(input.InstanceType))
-    return natsmsg.NATSRequest[ec2.Reservation](ctx, s.natsConn, topic, input, 5*time.Minute, accountID)
+func runIntoReservation(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, accountID, crID string) (*ec2.Reservation, error) {
+    subject := "ec2.RunInstances.cr." + crID
+    reservation, err := natsmsg.NATSRequest[ec2.Reservation](ctx, natsConn, subject, input, 5*time.Minute, accountID)
+    // ...
 }
 ```
 

docs/admin/setting-up-your-cluster/README.md

@@ -340,7 +340,7 @@ spx get vms
 Spinifex ships with a built-in web console — an alternative to the AWS CLI, SDKs, and Terraform, analogous to the AWS Management Console. Every action in the UI is a standard AWS SigV4 API call, so the same IAM policies and audit behaviour apply.
 
 <p align="center">
-  <img src="../../../.github/assets/spinifex-ui.jpg" alt="Spinifex Web UI" width="900">
+  <img src="../../../.github/assets/dashboard.webp" alt="Spinifex Web UI" width="900">
 </p>
 
 ### Open the Console

docs/containers/eks/README.md

@@ -59,9 +59,6 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM
 
 - **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
 - **The cluster security group is auto-managed.** As on EKS, Spinifex creates one `eks-cluster-sg-<name>-<id>` in your VPC, tagged `kubernetes.io/cluster/<name>=owned` and `aws:eks:cluster-name`, allowing all traffic from itself. It is attached to the workers and the private-endpoint network interface, and `DescribeCluster` reports it as `clusterSecurityGroupId`. A launch template that names security groups replaces it on the workers, as on AWS. Clusters created before this change keep their `-control-plane-sg` and `-nodegroup-sg` until you recreate them. Deleting a cluster deletes the cluster security group and removes only the rules on your other security groups that reference it. `vpc_config.security_group_ids` is ignored.
-- **Node group deletes are asynchronous.** `DeleteNodegroup` returns at once with the group `DELETING`, and `DescribeNodegroup` answers `ResourceNotFoundException` once the workers and the group's instance profile are gone. Deleting a cluster or creating a group of the same name while a group is `DELETING` returns `ResourceInUseException`, as does a second delete of a `DELETING` group.
-- **If a node group delete cannot finish,** the group moves to `DELETE_FAILED` after 30 minutes, with a `NODE_TERMINATION_FAILURE` health issue naming the workers that are still running. Run `DeleteNodegroup` again to resume it.
-- **Each node group owns an EKS-managed instance profile.** Spinifex creates it, tags it `spinifex:eks-cluster`, `spinifex:eks-nodegroup` and `spinifex:eks-profile-id`, and puts the node role in it. It is removed with the group, so you can delete the node role afterwards with no extra steps. Spinifex never deletes an instance profile it did not create, or one whose tags, name or role no longer match what it recorded; it leaves the profile and logs a warning with its name.
 - **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
 - **Workers launch with IMDSv2 required and an IMDS hop limit of 2.**
   This matches AWS managed node groups and lets a pod without IRSA reach the node role through the SDK default credential chain.
@@ -380,15 +377,6 @@ Control plane readiness and kubectl authentication above lists which clusters re
 
 **`create-cluster` rejected with an authentication-mode error.** `CONFIG_MAP` is the one mode Spinifex rejects, because it has no `aws-auth` ConfigMap path. Set `authentication_mode` to `API` or `API_AND_CONFIG_MAP`, or leave it unset, and grant access with access entries.
 
-**`delete-role` fails with `DeleteConflict` after deleting a node group.** Node groups created before per-group instance profiles left an instance profile named after the role, which Spinifex no longer removes because it cannot prove it created it. Remove it yourself:
-
-```bash
-aws iam remove-role-from-instance-profile --instance-profile-name <role-name> --role-name <role-name>
-aws iam delete-instance-profile --instance-profile-name <role-name>
-```
-
-**A node group is `DELETE_FAILED`.** `aws eks describe-nodegroup` lists the workers that did not stop under `health.issues`. Once they are gone, or after you fix the cause, run `aws eks delete-nodegroup` again.
-
 **Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, use the AWS form from `DescribeAddonVersions` (see Add-on versions above), or leave `addon_version` unset so Spinifex selects the catalog default.
 
 ## Reference Architectures

docs/oci-integration/README.md

@@ -633,7 +633,7 @@ ssh -i ~/.ssh/oci-spx ubuntu@"$(head -1 .validate-vm-multi/hosts)" \
 If this returns a list of available instance types, your installation is working.
 
 <p align="center">
-  <img src="../../.github/assets/spinifex-ui.jpg" alt="Spinifex Web UI" width="900">
+  <img src="../../.github/assets/dashboard.webp" alt="Spinifex Web UI" width="900">
 </p>
 
 

docs/package-boundary/eks-cluster-lifecycle.md

@@ -34,7 +34,7 @@ Node group workers and CSI-provisioned EBS volumes are never removed by a cluste
 
 ## Gaps
 
-1. Three `eks.*` handlers still hold their request for work that grows with the resource, and the subject table names them as not conformant: `CreateCluster` (reclaiming a `FAILED` cluster of the same name tears its infra down inline), `UpdateNodegroupConfig` (one awaited launch or terminate per worker added or removed) and `RestoreSnapshot` (launches a control-plane VM and fences the old one inline).
+1. Four `eks.*` handlers still hold their request for work that grows with the resource, and the subject table names them as not conformant: `CreateCluster` (reclaiming a `FAILED` cluster of the same name tears its infra down inline), `DeleteNodegroup` (one awaited terminate per worker), `UpdateNodegroupConfig` (one awaited launch or terminate per worker added or removed) and `RestoreSnapshot` (launches a control-plane VM and fences the old one inline).
 2. AWS's answer to a repeat delete on a `DELETING` cluster has not been observed; `DELETING` from the record is the chosen behaviour.
 3. Control-plane, load balancer and VPC resources in the system account are keyed by the bare cluster name, so a teardown by name can reach another tenant's same-named cluster.
 4. The teardown deletes load balancers the in-cluster controller created in the customer account, which AWS leaves behind.

docs/security/network-connections/README.md

@@ -99,6 +99,7 @@ behavior in other words.
 | 4432 | Formation server | HTTPS | Cluster (bootstrap only) | Cluster join coordination; active only while a join token is valid. Binds the node's `--bind` (lan) address. See *Formation port lifecycle* below. | Short-lived bearer token + TLS¹ |
 | 4222 | spinifex-nats (client) | NATS + TLS | Cluster | Internal service bus for EC2/EBS/VPC/S3 handlers | Token + mutual TLS (cluster CA) |
 | 4248 | spinifex-nats (cluster) | NATS + TLS | Cluster | Inter-node NATS federation | Token + mutual TLS (cluster CA) |
+| 5001 | iperf2 (admin network test) | TCP | Cluster, any plane | Throughput test between two nodes, started only by the super-admin account. Listens on one plane address for the length of one test, then exits. Runs with no capabilities. | None — peer-scoped by firewall, and the client may only target its own plane subnet |
 | 5300 | northstar | DNS (UDP + TCP) | Cluster | Forward target for every node's per-instance DNS shim, dialled cross-node. Binds the wildcard by design. | None — restrict by firewall |
 | 6660 | predastore (blob node) | QUIC / UDP | Cluster | Erasure-coded object shard transport between hosts. Multi-node clusters only — see *Predastore ports* below. | TLS 1.3, server certificate verified against the cluster CA |
 | 7660 | predastore (meta node) | QUIC / UDP | Cluster | Raft consensus over global state — buckets and the object index — between hosts. Multi-node clusters only. | TLS 1.3, server certificate verified against the cluster CA |
@@ -221,7 +222,7 @@ table inet spinifex_filter {
 
     # Cluster, from peer nodes only. Replace with your nodes' lan-plane
     # addresses — a CIDR does not generalise to nodes on different subnets.
-    ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
+    ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } tcp dport { 4222, 4248, 4432, 5001, 5300, 6641, 6642, 6643, 6644, 8660 } accept
     ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } udp dport { 5300, 6660, 7660 } accept
 
     # Encap, from peer chassis only. Replace with your nodes' vpc-plane

docs/ui/design-system.md

@@ -24,7 +24,7 @@ All colours are OKLCH and come from mulgadc.com, so product and site match.
 | Token | Use |
 |---|---|
 | `background` | The app page. Warm off-white in light, ink `#14181D` in dark. |
-| `card`, `popover` | Raised surfaces. White in light, about `#1B2027` in dark. |
+| `card`, `popover` | Raised surfaces. White in light, about `#1B2027` in dark. `popover` is every floating surface: tooltips, popovers, menus and pickers. |
 | `muted`, `secondary`, `accent` | Hover rows, secondary buttons, quiet fills. |
 | `border` | Every divider and card edge. `#DCD8CE` in light, `oklch(1 0 0 / 8%)` in dark. |
 | `brand` | Brand orange `#EE4B12`, for the logo and decoration only. It fails text contrast, so it never carries text. |
@@ -39,15 +39,22 @@ Each status has a solid token plus `-foreground` (text on a tinted background),
 
 | Status | Hue | AWS states |
 |---|---|---|
-| `success` | green | running, available, completed, healthy |
-| `warning` | brand yellow `#F5B500` | pending, stopping, creating, modifying, deleting, draining |
-| `danger` | cool red, hue 17 | stopped, error, failed, unhealthy, unavailable |
+| `success` | green | settled and good: running, available, active, associated, attached, in-use, healthy, ok, main |
+| `warning` | brand yellow `#F5B500` | changing state and will settle: pending, stopping, creating, modifying, deleting, draining |
+| `caution` | orange, hue 58 | works but needs attention: degraded, impaired, insufficient-data |
+| `danger` | cool red, hue 17 | stopped, error, failed, update-failed, unhealthy, unavailable, blackhole |
 | `info` | blue | informational notes |
-| `neutral` | grey `#9AA1A8` | terminated (muted text), and any unmapped state |
+| `neutral` | grey `#9AA1A8` | gone or absent (terminated, deleted, no) in muted text, and any unmapped state |
 
 `stopped` is danger because EC2 reads it that way, although RDS treats it as a resting state; splitting the map by service is its own change.
 
 `StateBadge` (`src/components/state-badge.tsx`) is the only mapping from an AWS state to a tone, and it exposes the result as `data-tone` for tests.
+Its tones are `success`, `progress` (the yellow `warning` token), `caution`, `danger`, `muted` and `neutral`.
+A `progress` state carries the submit button's spinning loader, so a launching instance reads as work under way; `StatusBadge` does not, because its yellow can mean waiting on the user, as a zone's `pending nameservers` does.
+It shows every state lowercase with hyphens, as EC2 writes them, so EKS's `ACTIVE` reads `active` and Bedrock's `IN_PROGRESS` reads `in-progress`; the API value, filters and CLI panels keep AWS's case.
+A good or settled state is always green, never grey, and a status that is not an API state, such as a cluster's health, uses `StatusBadge` with a tone rather than a `Badge`.
+The orange of `caution` is kept clear of the primary accent's hue, which marks actions and never a status.
+`Badge`'s default variant is the brand primary and is never used for a status; a label that is not a status, such as `default v3` or `Leader`, uses a `secondary` or `outline` `Badge`.
 Every status badge measures above 4.8:1 against its tinted background in both themes.
 
 **Danger is separated from the primary accent by hue alone in dark mode**: the two have the same lightness.
@@ -67,18 +74,21 @@ Colour is never the only key: label every series.
 | UI text, tables and forms | Geist (`font-sans`) |
 | IDs, ARNs, IPs, CIDRs, code and eyebrow labels | JetBrains Mono (`font-mono`), with `tabular-nums` for figures |
 
-| Step | Size | Use |
-|---|---|---|
-| `2xl` | 26px | page title (`PageHeading`) |
-| `xl` | 22px | section heading |
-| `lg` | 17px | |
-| `base` | 15px | |
-| `sm` | 14px | body and table default |
-| `xs` | 13px | badges, secondary text |
-| `2xs` | 12px | mono eyebrow labels |
+| Step | rem | At the default 16px browser font | Use |
+|---|---|---|---|
+| `2xl` | 1.625 | 28.6px | page title (`PageHeading`) |
+| `xl` | 1.375 | 24.2px | section heading |
+| `lg` | 1.0625 | 18.7px | |
+| `base` | 0.9375 | 16.5px | |
+| `sm` | 0.875 | 15.4px | body and table default |
+| `xs` | 0.8125 | 14.3px | badges, secondary text |
+| `2xs` | 0.75 | 13.2px | mono eyebrow labels |
 
-The scale sits a step above Tailwind's defaults, as consoles such as Vultr's do, because 13px body text read as cramped on a wide screen.
-Code blocks are set at 12px (`--text-code-block`) so a long command still fits.
+The root font size is 110% (`html` in `src/styles.css`), so one rem is 17.6px at the browser default.
+Everything is sized in rem, so text, spacing and breakpoints scale together, as they would at 110% browser zoom, and a user's own font-size setting still applies.
+The scale sits above Tailwind's defaults, as consoles such as Vultr's do, because the console read as small on a wide screen at 100%.
+Code blocks are set at 0.75rem (`--text-code-block`) so a long command still fits.
+Size new things in rem through the type and spacing scales, never in px, or they will not scale with the rest.
 
 ### Primary and contrast
 
@@ -169,6 +179,22 @@ Every other list is a `DataTable`, and the test also fails when an allowed file
 | S13 | a component outside `ui/` renders a raw `<input>`, `<select>` or `<textarea>` |
 | S15 | a route imports the `ui/alert` primitive instead of using `ErrorBanner` |
 
+Rules that ADR-0016 does not yet number are named by what they hold until their amendment is accepted:
+
+| Rule | Fails when |
+|---|---|
+| status pills come from StateBadge or StatusBadge | a component outside `ui/` renders `<Badge>` with the default variant |
+| status pills come from StateBadge or StatusBadge | a file outside `state-badge.tsx` builds a `rounded-full` pill on a status colour |
+| floating surfaces follow the theme | a file fills with `bg-foreground` or `text-background`, or a tooltip, popover, hover card, menu, select or combobox is not on `bg-popover` and `text-popover-foreground` |
+| expanding a panel keeps its tab | a `fullPage` link does not pass `search={{ tab }}` |
+| resource tags are edited in place | `PanelTags` or `TagsEditor` comes back |
+| lists filter through one FilterBar | a component other than `FilterBar` imports `ui/toggle-group` |
+| a component handed a form reads errors through `useFormState` | a file typed with `UseFormReturn` reads `form.formState` |
+
+The last one matters because the console builds with the React compiler and the unit tests run without it.
+With the compiler on, a child handed a react-hook-form object is cached on that object, whose identity never changes, so errors read from `form.formState` never draw and the submit looks dead.
+`useFormState({ control })` subscribes the child itself.
+
 An exception is named in that rule's allowlist with its reason, and the test fails when an allowed file stops needing it.
 S11, S14 and S16 are held by review, since none has a shape in the source a test can read without false alarms.
 
@@ -177,7 +203,7 @@ S11, S14 and S16 are held by review, since none has a shape in the source a test
 `src/lib/nav.ts` is the one description of the console's navigation.
 The sidebar, the command palette and the breadcrumbs all read it, and `nav.test.ts` checks every destination against the generated route tree, so a renamed route fails a test rather than leaving a dead link.
 
-- **Sidebar.** Groups take generic names rather than AWS's service marks: Compute, Databases, Authentication and Object Storage, not EC2, RDS, IAM and S3. Pages and the rest of the console keep the AWS names, and each group's `service` keeps its AWS name findable in the command palette, where typing it exactly ranks that group first. Each service group folds, and a folded group stays folded across reloads (`localStorage["spinifex-ui-nav-collapsed"]`). Navigating into a folded group unfolds it so the active item shows, and it can then be folded again, the current group included. On a phone the sidebar is a sheet, and any navigation closes it.
+- **Sidebar.** Groups take generic names rather than AWS's service marks: Compute, Databases, Authentication and Object Storage, not EC2, RDS, IAM and S3. Pages and the rest of the console keep the AWS names, and each group's `service` keeps its AWS name findable in the command palette, where typing it exactly ranks that group first. Each service group folds, and a folded group stays folded across reloads (`localStorage["spinifex-ui-nav-collapsed"]`). Navigating into a folded group unfolds it so the active item shows, and it can then be folded again, the current group included. Folding animates through the `nav` variant of `CollapsibleContent`, opening over 200ms and closing over 150ms, and switches at once under reduced motion. On a phone the sidebar is a sheet, and any navigation closes it.
 - **Command palette.** ⌘K or Ctrl K, or the Search button in the header. It lists every page the user can reach, every create flow (`create` on a nav item), and a theme toggle. It is navigation only, with no resource search yet.
 - **Breadcrumbs.** Service, then list page, then the resource or the create flow. A resource ID is set in mono. On a phone only the last crumb shows.
 - **Sign in.** The sign-in card sits over the shell on a light, barely blurred overlay rather than on a page of its own, so the console stays in view. The shell behind it is `inert`, which keeps pointer, keyboard and screen reader out of it until the user signs in.
@@ -202,8 +228,13 @@ So columns drop out when a side panel narrows the list as well as on a small scr
 List pages keep their filter, state and sort in the URL, so a filtered view can be bookmarked, shared and survives a reload.
 EC2 Instances is the reference page (`ec2/(instances)/-components/instances-page.tsx`):
 
-- The first column names the resource: the `Name` tag in the primary colour, with the ID in mono beneath it, opening its side panel.
+- The first column is the instance ID in mono, opening its side panel, and the second its `Name` tag.
+- A long value ends in an ellipsis and shows in full on hover (`TruncatedText`), so it never widens its column.
+- A time column shows a relative time, `33 min ago` or `8 months ago`, with the full timestamp on hover (`RelativeTime`).
+- A Tags column shows one tag key as a pill and `+N` for the rest, and hover lists every tag with its value (`TagPills`); it leaves out `Name`, which has its own column.
+- Below `md`, only the ID, Name and State columns remain.
 - Bulk actions appear above the table once a row is selected. Each action counts only the selected instances it applies to, so a mixed selection never sends a call that must fail, and Terminate asks first and lists every ID.
+- Search and state filters sit in one `FilterBar` (`src/components/filter-bar.tsx`) above the table: a search box, an optional set of choices with counts, and Clear filters while anything is filtered.
 - An empty account gets a call to action, and an empty filter result says the filters are why.
 
 VPCs and Internet gateways follow the same pattern.
@@ -219,6 +250,12 @@ Every list with a delete action has row checkboxes. Selecting a row shows `Selec
 - **The bulk delete and the panel's delete are the same function.** Where a single delete is several calls, such as removing dependants first, the sequence lives in one place and both use it.
 - The verb follows the AWS console: Release for an Elastic IP, Deregister for an image.
 
+### Lists inside a panel
+
+A list inside a panel tab, such as a hosted zone's records, is a `DataTable` with row checkboxes, and a click on a row opens it for editing in a `FormPanel`.
+Rows the user cannot change, such as a zone's own SOA and NS, are muted and cannot be selected (`isRowActionable`).
+A batch delete uses the service's batch call where it has one, so Route 53 removes the whole selection in one `ChangeResourceRecordSets`.
+
 ### Typed confirmation
 
 A delete that cannot be walked back asks the user to type a word first, as the AWS console does before deleting a volume.
@@ -264,13 +301,24 @@ Modelled on UniFi's console: a click on a row opens the resource in a panel besi
 - **URL.** The panel is search state on the list route: `selected=<id>`, `tab=overview|monitoring|settings`, and `create=true`. Defaults are stripped, so a closed panel leaves the plain list address. Opening or closing a panel pushes a history entry, so Back closes it; a tab or filter change replaces the entry.
 - **Layout.** From `lg` (1024px) the panel is fixed to the right edge, full height below the header, flush with no outer margin, and it slides in over the list. The list keeps its width and columns underneath, so nothing reflows when a panel opens, and a soft gradient on the panel's left edge reads as the panel lying on top. Below `lg` there is no room to share, so the panel replaces the list as a full page with a back arrow. `useWideLayout` decides, through `matchMedia`.
 - **`DetailPanel`.** A header with the resource's own page (left) and close (right), the title and its ID, a summary of state and quick actions, then icon tabs with tooltips. Escape closes it unless a dialog or menu is open over it. Closing slides the panel out over 150ms, or at once under reduced motion. Each tab, and a create form, sits in its own error boundary, so a failed query shows in the panel and the list keeps working.
-- **Contents.** `PanelSection` is a folding card, `PanelFields` and `PanelField` a label and value list, with `copy` for a one-click copy of IDs and addresses, and `PanelTags` the tag list. Create forms take an `onClose(id?)` prop: embedded, they hide their back link and heading, and a create selects the new resource.
+- **Contents.** `PanelSection` is a folding card, `PanelFields` and `PanelField` a label and value list, with `copy` for a one-click copy of IDs and addresses, and `InlineTags` the tag list. Create forms take an `onClose(id?)` prop: embedded, they hide their back link and heading, and a create selects the new resource.
 - **One component is both the panel and the page.** Each resource has a single details component (`InstanceDetails`, `VpcDetails`, `InternetGatewayDetails`) that takes `layout="auto" | "page"`. The list renders it as the panel; `/ec2/describe-instances/$id` and its kind render the same component as a page, as one centred column under a standard left-aligned heading, with state and actions on the right and text tabs. A feature added to the panel is on the page by construction, and the page ignores Escape and has a back arrow in place of close. Expanding the panel to its page keeps the tab, and query results such as fetched console output come along from the cache.
 - **A click on the list's background closes the panel**, as UniFi's does. A row selects another resource instead, and links, buttons and inputs keep their own behaviour. Selecting rows for a bulk action closes it too, so the bulk bar and the panel never compete.
 - **Actions.** State actions sit in the summary as a button group, and the rest go in an Actions menu, Unikraft style. Stop, Reboot and every destructive action confirm in a dialog.
 - **Loading.** A panel or page still loading shows `PanelSpinner`, a centred spinner on the background, not a grey skeleton block.
 - **`DetailCard`, `DetailRow` and `ListCard` are retired.** Every resource page uses the panel sections instead.
 
+### Tags
+
+`InlineTags` (`src/components/inline-tags.tsx`) shows a resource's tags in its panel and edits them in place.
+Each row has a pencil, which turns the key and value into inputs and becomes a tick to save, and a bin to remove the tag; `+ Add tag` appends a row being edited.
+Enter saves and Escape cancels the row without closing the panel.
+Every row saves on its own request through the service's own tagging call, and a failure shows beside its row.
+AWS has no rename, so changing a key puts the new key and then removes the old one, and a failure between the two leaves both rather than neither.
+Keys are 1 to 128 characters with no `aws:` prefix and no duplicates, and values up to 256.
+Without `onPut` and `onRemove` it is read-only, as for a system image or an AWS-managed policy.
+Panels use the per-service wrappers in `src/components/resource-tags.tsx` and the writers in `src/mutations/tags.ts` rather than wiring the calls themselves.
+
 ### Forms from a resource's panel
 
 A form that acts on an open resource, such as attaching a volume, scaling a node group or adding a listener, opens in `FormPanel` (`src/components/form-panel.tsx`), never in a dialog.
@@ -285,8 +333,16 @@ Every create flow opens in the side panel over its list, through `CreatePanel`,
 
 - **URL.** `create=true` opens it, plus a list-local field where the flow needs context (`volume` for a snapshot, `instance` for an RDS snapshot, `launch=service|task` on ECS clusters). The old `create-*` addresses redirect to the list with the panel open, so links and bookmarks keep working.
 - **Form.** The fields come in the order AWS asks for them. Settings that are on or off are `Switch`es, and a multi-select picker (subnets, security groups) is a `SwitchList`, a boxed list with one switch per row. Checkboxes only select table rows.
-- **CLI.** Every create form ends with a `CliCommandPanel` showing the AWS CLI commands that send the same request, built from the form's live values and only the flags the console actually sends.
-- **Submit.** `FormActions` turns the submit button into a spinner with its pending label in place, and `FormLock`, a native `<fieldset disabled>`, disables every control above it until the request settles. Nothing above the form appears or moves, and the panel does not scroll.
+- **CLI.** Every create form ends with a `CliCommandPanel` showing the AWS CLI commands that send the same request, built from the form's live values and only the flags the console actually sends. The commands wrap to the panel's width and never scroll sideways; Copy takes the unwrapped text.
+- **Sections.** Related fields are grouped in `PanelSection` boxes, as the instance launch form groups Image, Instance, Network and Root volume.
+- **Nothing to pick, no field.** An optional picker with nothing usable to choose, such as Launch Template or Placement Group with none available, is left out of the form rather than shown holding only `none`.
+- **Advanced settings.** Settings most users leave at their defaults sit in a collapsed Advanced settings section, which opens by itself when a field inside it fails validation.
+- **Choices with a mark.** A few named products, such as database engines or distributions, are `ChoiceTile`s with the product's own mark from `src/assets/`, not a select. One that cannot be used stays visible and disabled, saying why.
+- **Durations and windows.** A duration is a select of presets with the AWS default first and named (`Auto (5 min)` for a record's TTL), keeping a custom value the API returned. A time window, such as an RDS backup or maintenance window, is a day, start time and length picker that refuses an overlap before submitting.
+- **Labels by type.** Where one field means different things by type, its label follows the type, as a record's value reads `IPv4 address` for A and `Mail server` for MX.
+- **Long pickers.** A `SwitchList` that can run long, such as security groups, scrolls inside its box rather than pushing the form down.
+- **Help.** A field's longer explanation goes in an (i) beside its label, through the `info` prop of `TextField` and `SelectField`, rather than in text under the field. Text under a field is for what the user must know before submitting.
+- **Submit.** `FormActions` cross-fades the submit label into a spinner with its pending label over 150ms, at one width, so the button never changes size, and `FormLock`, a native `<fieldset disabled>`, disables every control above it until the request settles. Nothing above the form appears or moves, and the panel does not scroll.
 - **Failure.** A refused request shows beside the button through `FormActions`'s `error`, and the form unlocks. The handler wraps only the mutation call in `try`, with an empty `catch` that returns, because react-hook-form rethrows whatever the handler throws and the error is already on screen from the mutation's state.
 - **Success.** `onClose(id)` closes the panel and selects the new resource, where the list has a panel to select it in.
 - **Appliance images.** A create that needs an ECS, EKS or RDS appliance image shows the missing-image callout inside its panel, never on the list, and only the step that needs the image is blocked.
@@ -306,6 +362,7 @@ A gap of more than three sample periods breaks the line rather than drawing acro
 ## Dates
 
 `formatDateTime` writes the AWS console's form in the viewer's zone, `2026/06/04 02:00 GMT-5`, with a 24-hour clock and UTC as plain `GMT`.
+In a table the time is relative (`relativeTime` in `src/lib/relative-time.ts`), rounded down to the largest whole unit, and the full form shows on hover.
 
 ## Testing
 

docs/vultr-integration/README.md

@@ -255,7 +255,7 @@ The same profile is set up on node 1, if you prefer to work from there:
 If this returns a list of available instance types, your installation is working.
 
 <p align="center">
-  <img src="../../.github/assets/spinifex-ui.jpg" alt="Spinifex Web UI" width="900">
+  <img src="../../.github/assets/dashboard.webp" alt="Spinifex Web UI" width="900">
 </p>
 <p align="center"><em>The Spinifex web console, on port 3000 of node 1.</em></p>
 

go.mod

@@ -27,10 +27,10 @@ require (
 	github.com/klauspost/cpuid/v2 v2.4.1
 	github.com/miekg/dns v1.1.73
 	github.com/minio/highwayhash v1.0.4
-	github.com/mulgadc/bluebottle v1.23.0
-	github.com/mulgadc/northstar v1.23.0
-	github.com/mulgadc/predastore v1.23.0
-	github.com/mulgadc/viperblock v1.23.0
+	github.com/mulgadc/bluebottle v1.24.0
+	github.com/mulgadc/northstar v1.24.0
+	github.com/mulgadc/predastore v1.24.0
+	github.com/mulgadc/viperblock v1.24.0
 	github.com/nats-io/nats-server/v2 v2.15.2
 	github.com/nats-io/nats.go v1.55.0
 	github.com/nats-io/nkeys v0.4.16

go.sum

@@ -301,14 +301,14 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/bluebottle v1.23.0 h1:h2BOSBDrNjIHA7QUeWX9xci2zcIHWsB1Ky0Q///QPaI=
-github.com/mulgadc/bluebottle v1.23.0/go.mod h1:eWfnT7nGgG93oChqMoTlGv2onPVZ2bVGSyg4nKl1s7I=
-github.com/mulgadc/northstar v1.23.0 h1:5oeFDB3fifzFb+I5nDAXsewt2rYdEbrUsHl7gwxPZ5k=
-github.com/mulgadc/northstar v1.23.0/go.mod h1:zZtrmR8ITOhZZQrVbCJQUBpOUiblA2kogKh0Oi5xh/8=
-github.com/mulgadc/predastore v1.23.0 h1:JpjhDyF1CLTLGDmXrYPSFRImDXfu5L2zeXdiHPHwtNw=
-github.com/mulgadc/predastore v1.23.0/go.mod h1:y5nBVTX+E+6G2z9xkH9gkecglKbUGnCCitd7d8ZZaJ8=
-github.com/mulgadc/viperblock v1.23.0 h1:/HZDVVw6/tiq497AVjMeEG+dLn4du4pcbBSiwZKCamg=
-github.com/mulgadc/viperblock v1.23.0/go.mod h1:frYnDTsFF0MTl7dP84fcO4PsyWvHMb8EQQFIPajxai0=
+github.com/mulgadc/bluebottle v1.24.0 h1:7rwzW4n0lo3KWGpuUYiQ8KhIFNV/4o1AChrEp79PCKE=
+github.com/mulgadc/bluebottle v1.24.0/go.mod h1:BzyzOpzpzAyKhGYppwcOIEQRqMASc6IoKCBhaaODXbQ=
+github.com/mulgadc/northstar v1.24.0 h1:2RtBUCMNvwmkqTlTi1Ve5/9Hkqe6gWWUVMkSQ0nhbhY=
+github.com/mulgadc/northstar v1.24.0/go.mod h1:0NfFxVWt2ZZQzZ7ObfsEJ2O7lL1yjKKf3ZGF4poH0QM=
+github.com/mulgadc/predastore v1.24.0 h1:YECc+vOmZRQYaMMjKmJOnFcKZ7cOgTr3b7Sb4JmgmDU=
+github.com/mulgadc/predastore v1.24.0/go.mod h1:+EKnT7S5POYhUPE6ZS5mvMR6TS735P0m9UsOrQ88TAg=
+github.com/mulgadc/viperblock v1.24.0 h1:i4WNGvfnDeYppqohWDH5qGpUrchl4qpTqyQxj/BV7nk=
+github.com/mulgadc/viperblock v1.24.0/go.mod h1:iY4bdiXYdGbxBrJe1zizGGh65yBGEta9CWlIhpgHne0=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=

scripts/build-lambda-runtime-microvm-image.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+# build-lambda-runtime-microvm-image.sh — build the Lambda runtime microVM kernel
+# + initramfs: a guest running the Lambda guest agent, with virtio-net for its
+# host control connection. It is not shipped.
+#
+# Inputs:
+#   $REPO_ROOT/build/microvm/lambda-runtime/{init.sh,inittab,run-agent}
+#   $REPO_ROOT/bin/lambda-agent  (make bin/lambda-agent)
+#
+# Outputs:
+#   $REPO_ROOT/build/microvm/lambda-runtime/vmlinuz
+#   $REPO_ROOT/build/microvm/lambda-runtime/initramfs.cpio.gz
+#
+# Rootfs strategy and module stripping: scripts/lib/microvm-image.sh.
+# shellcheck disable=SC2034 # variables are consumed by the sourced library
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+BUILD_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
+INIT_SH="$BUILD_DIR/init.sh"
+INITTAB="$BUILD_DIR/inittab"
+RUN_AGENT="$BUILD_DIR/run-agent"
+LAMBDA_AGENT_BIN="$REPO_ROOT/bin/lambda-agent"
+TAG="[build-lambda-runtime-microvm-image]"
+
+ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
+ALPINE_MIRROR="${ALPINE_MIRROR:-https://dl-cdn.alpinelinux.org/alpine}"
+# kmod: the image's own depmod/modprobe rebuild module metadata. busybox ships ip.
+ALPINE_PACKAGES="busybox linux-virt kmod ca-certificates"
+# tiny_power_button turns the ACPI power button into a signal busybox init
+# handles as poweroff.
+MICROVM_EXTRA_MODULES="${MICROVM_EXTRA_MODULES:-tiny_power_button}"
+# virtio_net needs failover and net_failover, which the virtio* glob misses.
+MICROVM_EXTRA_KEEP_GLOBS=("*failover*.ko*")
+MICROVM_GATE_MODULES="qemu_fw_cfg virtio_net"
+
+# shellcheck source=lib/microvm-image.sh
+source "$REPO_ROOT/scripts/lib/microvm-image.sh"
+
+echo "$TAG repo root: $REPO_ROOT"
+echo "$TAG build dir: $BUILD_DIR"
+
+microvm_check_tools
+
+for f in "$INIT_SH" "$INITTAB" "$RUN_AGENT"; do
+    if [ ! -f "$f" ]; then
+        echo "ERROR: $f not found — cannot build initramfs" >&2
+        exit 1
+    fi
+done
+if [ ! -x "$LAMBDA_AGENT_BIN" ]; then
+    echo "ERROR: $LAMBDA_AGENT_BIN not found — run 'make bin/lambda-agent'" >&2
+    exit 1
+fi
+if ! file -b "$LAMBDA_AGENT_BIN" | grep -q "statically linked"; then
+    echo "ERROR: $LAMBDA_AGENT_BIN is not statically linked — it cannot run on the musl guest" >&2
+    exit 1
+fi
+
+microvm_init_dirs
+microvm_make_rootfs
+microvm_empty_dev
+
+echo "$TAG installing init, inittab, run-agent and lambda-agent..."
+install -m 0755 "$INIT_SH" "$CHROOT_DIR/init"
+install -m 0644 "$INITTAB" "$CHROOT_DIR/etc/inittab"
+mkdir -p "$CHROOT_DIR/usr/local/lib/lambda-runtime" "$CHROOT_DIR/usr/local/bin"
+install -m 0755 "$RUN_AGENT" "$CHROOT_DIR/usr/local/lib/lambda-runtime/run-agent"
+install -m 0755 "$LAMBDA_AGENT_BIN" "$CHROOT_DIR/usr/local/bin/lambda-agent"
+# Function-code mountpoint and scratch space ship empty.
+mkdir -p "$CHROOT_DIR/var/task" "$CHROOT_DIR/tmp" \
+    "$CHROOT_DIR/proc" "$CHROOT_DIR/sys" "$CHROOT_DIR/dev" "$CHROOT_DIR/etc/ssl/certs"
+# The bootstrap runs unprivileged, so / must be traversable (mktemp -d makes it
+# 0700) and nothing it runs from may be group-writable.
+chmod 0755 "$CHROOT_DIR" "$CHROOT_DIR/var/task" "$CHROOT_DIR/usr/local/lib/lambda-runtime"
+
+microvm_locate_kernel
+microvm_strip_modules
+microvm_strip_rootfs
+microvm_stage_kernel
+microvm_pack_initramfs
+
+# Nothing reaches $BUILD_DIR until the staged pair passes these checks.
+for out in vmlinuz initramfs.cpio.gz; do
+    if [ ! -s "$STAGE_DIR/$out" ]; then
+        echo "ERROR: staged $out is missing or empty" >&2
+        exit 1
+    fi
+done
+if ! gzip -t "$STAGE_DIR/initramfs.cpio.gz"; then
+    echo "ERROR: staged initramfs.cpio.gz is not valid gzip" >&2
+    exit 1
+fi
+mv -f "$STAGE_DIR/vmlinuz" "$BUILD_DIR/vmlinuz"
+mv -f "$STAGE_DIR/initramfs.cpio.gz" "$BUILD_DIR/initramfs.cpio.gz"
+echo ""
+echo "$TAG done."

scripts/build-sqs-store-microvm-image.sh

@@ -18,6 +18,7 @@
 #   1. apk available on host → install directly into a temp chroot (Alpine CI/CD)
 #   2. docker available      → run alpine container, export filesystem
 #   3. podman available      → same via podman
+# shellcheck disable=SC2034 # variables are consumed by the sourced library
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
@@ -44,21 +45,19 @@ ALPINE_PACKAGES="busybox linux-virt iproute2 ca-certificates e2fsprogs kmod"
 # button (QMP system_powerdown) into a signal busybox init handles as poweroff.
 MICROVM_EXTRA_MODULES="${MICROVM_EXTRA_MODULES:-ext4 tiny_power_button}"
 
+# Module file globs kept on top of virtio*/*fw_cfg*: virtio_net deps for live migration.
+MICROVM_EXTRA_KEEP_GLOBS=("*failover*.ko*")
+# Modules the image modprobe must resolve, beyond the extras.
+MICROVM_GATE_MODULES="qemu_fw_cfg virtio_net"
+
+# shellcheck source=lib/microvm-image.sh
+source "$REPO_ROOT/scripts/lib/microvm-image.sh"
+
 echo "$TAG repo root: $REPO_ROOT"
 echo "$TAG build dir: $BUILD_DIR"
 mkdir -p "$BUILD_DIR"
 
-# --- Verify non-negotiable tools ---
-# fakeroot is mandatory: device nodes are written into the cpio archive inside a
-# faked-root environment so no real device node is ever created on the host
-# filesystem (which previously required sudo mknod and risked clobbering host
-# /dev/null on cleanup).
-for tool in cpio gzip find fakeroot; do
-    if ! command -v "$tool" >/dev/null 2>&1; then
-        echo "ERROR: required tool not found: $tool" >&2
-        exit 1
-    fi
-done
+microvm_check_tools
 
 if [ ! -x "$SQS_WORKER_BIN" ]; then
     echo "ERROR: $SQS_WORKER_BIN not found — run 'make bin/sqs-worker'" >&2
@@ -90,100 +89,11 @@ if [ "$nats_server_version" != "$NATS_SERVER_PIN" ]; then
 fi
 echo "$TAG nats-server: $nats_server_version (matches go.mod)"
 
-# --- Create chroot ---
-CHROOT_DIR=$(mktemp -d)
-# Guard: every destructive op below targets "$CHROOT_DIR/...". An empty or
-# unexpected value would aim rm -rf at the host (e.g. /dev). Refuse anything that
-# is not a freshly-created temp dir.
-case "${CHROOT_DIR:?mktemp -d returned empty}" in
-    /tmp/*|/var/tmp/*|"${TMPDIR:-/nonexistent}"/*) ;;
-    *) echo "ERROR: refusing to use unexpected CHROOT_DIR: $CHROOT_DIR" >&2; exit 1 ;;
-esac
-# Outputs are written here first, beside $BUILD_DIR so publishing is a rename.
-STAGE_DIR=$(mktemp -d "$BUILD_DIR/.stage.XXXXXX")
-CONTAINER_TOOL=""
-CONTAINER_CID=""
-cleanup() {
-    echo "$TAG cleaning up chroot: $CHROOT_DIR"
-    rm -rf "$CHROOT_DIR"
-    rm -rf "${STAGE_DIR:?}"
-    if [ -n "$CONTAINER_CID" ] && [ -n "$CONTAINER_TOOL" ]; then
-        echo "$TAG cleaning up container: $CONTAINER_CID"
-        $CONTAINER_TOOL rm -f "$CONTAINER_CID" >/dev/null 2>&1 || true
-    fi
-}
-trap cleanup EXIT
+microvm_init_dirs
 
-# --- Populate rootfs ---
-# Strategy 1: native apk (Alpine hosts, CI)
-if command -v apk >/dev/null 2>&1; then
-    echo "$TAG rootfs: native apk"
-    mkdir -p \
-        "$CHROOT_DIR/etc/apk" \
-        "$CHROOT_DIR/lib/apk/db" \
-        "$CHROOT_DIR/var/cache/apk"
-    cat > "$CHROOT_DIR/etc/apk/repositories" <<EOF
-${ALPINE_MIRROR}/v${ALPINE_VERSION}/main
-${ALPINE_MIRROR}/v${ALPINE_VERSION}/community
-EOF
-    apk add \
-        --root "$CHROOT_DIR" \
-        --arch x86_64 \
-        --no-cache \
-        --allow-untrusted \
-        --initdb \
-        $ALPINE_PACKAGES
+microvm_make_rootfs
 
-# Strategy 2: docker -> BuildKit local export.
-# Do NOT use `docker export` here. On Docker's overlay2 driver it leaks the
-# container's overlay mount (the mount survives `docker rm`), orphaning layers
-# under /var/lib/docker that only a daemon restart reclaims; on a busy CI host
-# this grows without bound. `buildx --output type=local` writes the image
-# filesystem straight to a directory and releases every mount, so nothing leaks.
-elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
-    echo "$TAG rootfs: docker buildx --output type=local (alpine:${ALPINE_VERSION})"
-    BUILD_CTX=$(mktemp -d)
-    cat > "$BUILD_CTX/Dockerfile" <<DOCKERFILE
-FROM alpine:${ALPINE_VERSION}
-RUN apk add --no-cache ${ALPINE_PACKAGES}
-DOCKERFILE
-    docker buildx build \
-        --platform linux/amd64 \
-        --output "type=local,dest=${CHROOT_DIR}" \
-        "$BUILD_CTX"
-    rm -rf "$BUILD_CTX"
-
-# Strategy 3: podman -> run + export. Podman's store does not exhibit the docker
-# overlay2 export-mount leak, and podman has no buildx.
-elif command -v podman >/dev/null 2>&1; then
-    echo "$TAG rootfs: podman export (alpine:${ALPINE_VERSION})"
-    CONTAINER_TOOL=podman
-    CONTAINER_CID=$(podman run -d \
-        "alpine:${ALPINE_VERSION}" \
-        sh -c "apk add --no-cache ${ALPINE_PACKAGES}")
-
-    exit_code=$(podman wait "$CONTAINER_CID")
-    if [ "$exit_code" != "0" ]; then
-        echo "ERROR: package installation failed in container (exit $exit_code)" >&2
-        exit 1
-    fi
-
-    podman export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR"
-
-else
-    echo "ERROR: no rootfs build tool available." >&2
-    echo "       Install one of: apk (Alpine), docker, podman" >&2
-    exit 1
-fi
-
-# --- /dev: empty mountpoint only ---
-# tar extraction without root silently creates 0-byte regular files for device
-# nodes. Drop them and leave an empty /dev mountpoint. The two static nodes the
-# kernel needs (/dev/console to wire init's stdio, /dev/null) are written into
-# the cpio archive under fakeroot at pack time — NOT created on the host. init
-# mounts devtmpfs over /dev as its first action for everything else at runtime.
-rm -rf "${CHROOT_DIR:?}/dev"
-mkdir "$CHROOT_DIR/dev"
+microvm_empty_dev
 
 # --- Place init script ---
 echo "$TAG installing init.sh as /init..."
@@ -227,30 +137,8 @@ mkdir -p \
     "$CHROOT_DIR/dev" \
     "$CHROOT_DIR/etc/ssl/certs"
 
-# --- Locate kernel ---
-KERNEL_IMG=$(find "$CHROOT_DIR/boot" -name "vmlinuz*" | sort | tail -1)
-if [ -z "$KERNEL_IMG" ]; then
-    echo "ERROR: no vmlinuz found in $CHROOT_DIR/boot" >&2
-    exit 1
-fi
-echo "$TAG kernel image: $KERNEL_IMG"
 
-KERNEL_VER=$(basename "$KERNEL_IMG" | sed 's/vmlinuz-//')
-KERNEL_CONFIG="$CHROOT_DIR/boot/config-${KERNEL_VER}"
-
-# --- Assert fw_cfg module ---
-if ! find "$CHROOT_DIR/lib/modules" \
-        -name "qemu_fw_cfg.ko*" \
-        -o -name "fw_cfg_sysfs.ko*" \
-    | grep -q .; then
-    if ! grep -q "CONFIG_FW_CFG_SYSFS=y" "$KERNEL_CONFIG" 2>/dev/null; then
-        echo "ERROR: qemu_fw_cfg module missing from initramfs — fw_cfg reads will fail at boot" >&2
-        exit 1
-    fi
-    echo "$TAG fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
-else
-    echo "$TAG fw_cfg: module found in lib/modules"
-fi
+microvm_locate_kernel
 
 # --- Assert ext4 userspace tools (data volume) ---
 for fs_tool in mkfs.ext4 e2fsck; do
@@ -261,227 +149,10 @@ for fs_tool in mkfs.ext4 e2fsck; do
 done
 echo "$TAG e2fsprogs: mkfs.ext4 + e2fsck present"
 
-# --- Strip kernel modules (Phase 2 keep-list approach) ---
-# Only modules needed by a virtio-mmio microvm guest are kept:
-#   virtio*.ko*    — virtio bus, net, blk, rng, console, and helpers
-#   *fw_cfg*.ko*   — QEMU fw_cfg sysfs driver (delivers boot blobs)
-#   *failover*.ko* — net_failover + failover (virtio_net deps for live migration)
-#   $MICROVM_EXTRA_MODULES — ext4 by default, plus dependency closure
-# All other loadable modules are removed. Drivers compiled into the kernel
-# (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
-
-# module_path prints a module's kernel/ path from the stock modules.dep, by
-# name or modules.alias; it fails when the module is not loadable.
-module_path() {
-    local kdir="$1" name="${2//-/_}" path alias_target
-    path=$(awk -F: -v n="$name" '{ f = $1; sub(/.*\//, "", f); sub(/\.ko(\.gz|\.xz|\.zst)?$/, "", f);
-        gsub(/-/, "_", f); if (f == n) { print $1; exit } }' "$kdir/modules.dep")
-    if [ -z "$path" ] && [ -f "$kdir/modules.alias" ]; then
-        alias_target=$(awk -v n="$2" '$1 == "alias" && $2 == n { print $3; exit }' "$kdir/modules.alias")
-        [ -n "$alias_target" ] && [ "$alias_target" != "$2" ] && path=$(module_path "$kdir" "$alias_target")
-    fi
-    [ -n "$path" ] && printf '%s\n' "$path"
-}
-
-# module_builtin reports whether a module name is compiled into the kernel.
-module_builtin() {
-    local kdir="$1" name="${2//-/_}"
-    [ -f "$kdir/modules.builtin" ] || return 1
-    awk -v n="$name" '{ f = $0; sub(/.*\//, "", f); sub(/\.ko$/, "", f); gsub(/-/, "_", f);
-        if (f == n) { found = 1; exit } } END { exit !found }' "$kdir/modules.builtin"
-}
-
-# image_kmod runs the image's own kmod (depmod, modprobe) through its musl
-# loader, so module metadata matches the guest's modprobe without a host kmod.
-image_kmod() {
-    "$CHROOT_DIR/lib/ld-musl-x86_64.so.1" --library-path "$CHROOT_DIR/lib:$CHROOT_DIR/usr/lib" \
-        --argv0 "$1" "$CHROOT_DIR/bin/kmod" "${@:2}"
-}
-
-echo "$TAG stripping kernel modules to virtio+fw_cfg+${MICROVM_EXTRA_MODULES} only..."
-for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
-    kver="$(basename "$kver_dir")"
-    kernel_dir="$kver_dir/kernel"
-    [ -d "$kernel_dir" ] || continue
-
-    if [ ! -s "$kver_dir/modules.dep" ]; then
-        echo "ERROR: stock ${kver_dir}modules.dep missing — cannot resolve module dependencies" >&2
-        exit 1
-    fi
-
-    # Extra modules start the closure; the glob families are kept as found.
-    keep_list=$(mktemp)
-    : > "$keep_list"
-    for name in $MICROVM_EXTRA_MODULES; do
-        if ! path=$(module_path "$kver_dir" "$name"); then
-            if module_builtin "$kver_dir" "$name"; then continue; fi
-            echo "ERROR: module $name is neither loadable nor built into $kver" >&2
-            exit 1
-        fi
-        echo "$path" >> "$keep_list"
-    done
-
-    # Close over modules.dep hard dependencies and modules.softdep pre:/post:
-    # names until the set stops growing. A softdep must be loadable or built in.
-    while :; do
-        before=$(sort -u "$keep_list" | wc -l)
-        for path in $(sort -u "$keep_list"); do
-            awk -F': ' -v p="$path" '$1 == p { n = split($2, d, " "); for (i = 1; i <= n; i++) print d[i] }' \
-                "$kver_dir/modules.dep" >> "$keep_list"
-            mod=$(basename "$path"); mod="${mod%%.ko*}"; mod="${mod//-/_}"
-            [ -f "$kver_dir/modules.softdep" ] || continue
-            for soft in $(awk -v n="$mod" '{ m = $2; gsub(/-/, "_", m) }
-                    $1 == "softdep" && m == n { for (i = 3; i <= NF; i++) if ($i != "pre:" && $i != "post:") print $i }' \
-                    "$kver_dir/modules.softdep"); do
-                if soft_path=$(module_path "$kver_dir" "$soft"); then
-                    echo "$soft_path" >> "$keep_list"
-                elif ! module_builtin "$kver_dir" "$soft"; then
-                    echo "ERROR: softdep $soft of $mod is neither loadable nor built into $kver" >&2
-                    exit 1
-                fi
-            done
-        done
-        after=$(sort -u "$keep_list" | wc -l)
-        [ "$before" -eq "$after" ] && break
-    done
-
-    extra_names=$(sort -u "$keep_list" | while read -r p; do m=$(basename "$p"); m="${m%%.ko*}"; echo "${m//-/_}"; done)
-    (cd "$kver_dir" && find kernel -name "virtio*.ko*" -o -name "*fw_cfg*.ko*" \
-        -o -name "*failover*.ko*") >> "$keep_list"
-
-    # Copy the keeper set out, preserving each kernel/ relative path.
-    tmp_keep=$(mktemp -d)
-    while read -r rel; do
-        rel="${rel#kernel/}"
-        mkdir -p "$tmp_keep/$(dirname "$rel")"
-        cp "$kernel_dir/$rel" "$tmp_keep/$rel"
-    done < <(sort -u "$keep_list")
-    rm -f "$keep_list"
-
-    # Replace kernel module tree with the keeper set.
-    rm -rf "$kernel_dir"
-    mkdir -p "$kernel_dir"
-    # Restore each kept file into the same relative path.
-    (cd "$tmp_keep" && find . -name "*.ko*" | while read -r f; do
-        dest_dir="$kernel_dir/$(dirname "${f#./}")"
-        mkdir -p "$dest_dir"
-        cp "$f" "$dest_dir/"
-    done)
-    rm -rf "$tmp_keep"
-
-    # Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
-    # kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
-    # produces an EMPTY modules.dep from compressed modules — the guest's
-    # modprobe then no-ops and qemu_fw_cfg never loads. Plain .ko sidesteps the
-    # host-kmod dependency entirely and the guest needs no decompressor either.
-    find "$kernel_dir" -name "*.ko.gz" -exec gunzip -f {} +
-
-    # Independent closure gate: each extra-closure module's own .modinfo depends=
-    # must name a kept module or a built-in, so a resolver bug fails here.
-    kept_names=$(find "$kernel_dir" -name "*.ko" -exec basename {} .ko \; | tr '-' '_' | sort -u)
-    while read -r ko; do
-        ko_name=$(basename "$ko" .ko); ko_name="${ko_name//-/_}"
-        grep -qx "$ko_name" <<< "$extra_names" || continue
-        deps=$(tr '\0' '\n' < "$ko" | grep -a -m1 '^depends=' | cut -d= -f2 | tr ',' ' ' || true)
-        for dep in $deps; do
-            dep="${dep//-/_}"
-            if ! grep -qx "$dep" <<< "$kept_names" && ! module_builtin "$kver_dir" "$dep"; then
-                echo "ERROR: $(basename "$ko") depends on $dep, which is neither kept nor built in" >&2
-                exit 1
-            fi
-        done
-    done < <(find "$kernel_dir" -name "*.ko")
-    for name in $MICROVM_EXTRA_MODULES; do
-        if ! grep -qx "${name//-/_}" <<< "$kept_names" && ! module_builtin "$kver_dir" "$name"; then
-            echo "ERROR: required module $name missing from the stripped module tree" >&2
-            exit 1
-        fi
-    done
-    echo "$TAG extra module closure: $(echo "$extra_names" | tr '\n' ' ')"
-    echo "$TAG modules kept: $(echo "$kept_names" | tr '\n' ' ')"
-
-    # Rebuild module metadata with the image's own kmod, the binary the guest's
-    # modprobe runs, so no host depmod is needed. Stock indexes still name .ko.gz
-    # files, so they are removed first and a failed rebuild cannot ship them.
-    for f in dep dep.bin alias alias.bin softdep weakdep symbols symbols.bin \
-        builtin.bin builtin.alias.bin devname; do
-        rm -f "$kver_dir/modules.$f"
-    done
-    if ! image_kmod depmod -b "$CHROOT_DIR" "$kver"; then
-        echo "ERROR: image depmod failed for $kver" >&2
-        exit 1
-    fi
-    if [ ! -s "$kver_dir/modules.dep" ] || [ ! -s "$kver_dir/modules.dep.bin" ]; then
-        echo "ERROR: ${kver_dir}modules.dep or modules.dep.bin missing or empty after depmod" >&2
-        exit 1
-    fi
-    # Resolution gate: the image's modprobe must resolve every required module
-    # to plain .ko files present in the stripped tree.
-    for name in $MICROVM_EXTRA_MODULES qemu_fw_cfg virtio_net; do
-        module_builtin "$kver_dir" "$name" && continue
-        if ! plan=$(image_kmod modprobe -d "$CHROOT_DIR" -S "$kver" --show-depends "$name"); then
-            echo "ERROR: image modprobe cannot resolve $name for $kver" >&2
-            exit 1
-        fi
-        while read -r verb path _; do
-            [ "$verb" = "insmod" ] || continue
-            case "$path" in
-                "$CHROOT_DIR/lib/modules/$kver/kernel/"*.ko) [ -f "$path" ] && continue ;;
-            esac
-            echo "ERROR: modprobe plan for $name names $path, not a kept plain .ko" >&2
-            exit 1
-        done <<< "$plan"
-        echo "$TAG modprobe $name: $(awk '{ n = split($2, a, "/"); printf "%s ", a[n] }' <<< "$plan")"
-    done
-done
-
-# --- Strip package-manager artifacts (not needed at runtime) ---
-echo "$TAG stripping package manager artifacts..."
-rm -rf \
-    "$CHROOT_DIR/lib/apk" \
-    "$CHROOT_DIR/var/cache/apk" \
-    "$CHROOT_DIR/etc/apk" \
-    "$CHROOT_DIR/usr/share/man" \
-    "$CHROOT_DIR/usr/share/doc" \
-    "$CHROOT_DIR/usr/share/apk" \
-    "$CHROOT_DIR/usr/include" \
-    "$CHROOT_DIR/usr/lib/pkgconfig"
-
-# --- Strip debug symbols from binaries ---
-# nats-server in /usr/local/bin is left as built (-s -w), so the shipped bytes
-# are the ones whose version was checked above.
-echo "$TAG stripping debug symbols from binaries..."
-find "$CHROOT_DIR/usr/sbin" "$CHROOT_DIR/usr/bin" \
-     "$CHROOT_DIR/sbin" "$CHROOT_DIR/bin" \
-    -type f 2>/dev/null | while read -r bin; do
-    strip --strip-all "$bin" 2>/dev/null || true
-done
-
-# --- Copy vmlinuz out before stripping /boot from chroot ---
-VMLINUZ_OUT="$STAGE_DIR/vmlinuz"
-echo "$TAG copying kernel: $VMLINUZ_OUT"
-cp "$KERNEL_IMG" "$VMLINUZ_OUT"
-
-# Drop /boot from the chroot — vmlinuz, Alpine's stock initramfs-virt, System.map
-# and kernel config all live here (~25 MiB). None are needed at runtime: the
-# guest already has the kernel loaded by QEMU's -kernel, and Alpine's initramfs
-# is unused because we replace it with our own /init. Leaving /boot in the cpio
-# nearly doubles uncompressed initramfs size and forces a 512 MiB guest just to
-# survive decompression.
-rm -rf "${CHROOT_DIR:?}/boot"
-
-# --- Build initramfs ---
-INITRAMFS_OUT="$STAGE_DIR/initramfs.cpio.gz"
-echo "$TAG building initramfs: $INITRAMFS_OUT"
-# Create the static device nodes and pack the archive inside a single fakeroot
-# session: mknod/cpio see faked device entries and write real char-device
-# records into the cpio, while the host filesystem never gets an actual node.
-fakeroot sh -c '
-    cd "$1" || exit 1
-    mknod -m 600 dev/console c 5 1
-    mknod -m 666 dev/null    c 1 3
-    find . | cpio --quiet -o -H newc | gzip -9 > "$2"
-' _ "$CHROOT_DIR" "$INITRAMFS_OUT"
+microvm_strip_modules
+microvm_strip_rootfs
+microvm_stage_kernel
+microvm_pack_initramfs
 
 # --- Gate the staged pair, then publish it ---
 # Nothing reaches $BUILD_DIR until every gate passes, so a failed build cannot

scripts/lib/microvm-image.sh

@@ -0,0 +1,391 @@
+#!/bin/bash
+# microvm-image.sh — shared steps for the microVM image builders, sourced by
+# build-sqs-store-microvm-image.sh and build-lambda-runtime-microvm-image.sh.
+#
+# The caller sets before use:
+#   TAG                      log prefix
+#   BUILD_DIR                output directory; STAGE_DIR is created beside it
+#   ALPINE_VERSION, ALPINE_MIRROR, ALPINE_PACKAGES
+#   MICROVM_EXTRA_MODULES    modules kept by name, with their dependency closure
+#   MICROVM_GATE_MODULES     modules the image modprobe must resolve, beyond the extras
+#   MICROVM_EXTRA_KEEP_GLOBS bash array of module file globs kept on top of virtio*/*fw_cfg*
+# Functions set CHROOT_DIR, STAGE_DIR, KERNEL_IMG and KERNEL_VER for the caller.
+
+
+microvm_check_tools() {
+    # --- Verify non-negotiable tools ---
+    # fakeroot is mandatory: device nodes are written into the cpio archive inside a
+    # faked-root environment so no real device node is ever created on the host
+    # filesystem (which previously required sudo mknod and risked clobbering host
+    # /dev/null on cleanup).
+    for tool in cpio gzip find fakeroot; do
+        if ! command -v "$tool" >/dev/null 2>&1; then
+            echo "ERROR: required tool not found: $tool" >&2
+            exit 1
+        fi
+    done
+}
+
+microvm_init_dirs() {
+    # --- Create chroot ---
+    CHROOT_DIR=$(mktemp -d)
+    # Guard: every destructive op below targets "$CHROOT_DIR/...". An empty or
+    # unexpected value would aim rm -rf at the host (e.g. /dev). Refuse anything that
+    # is not a freshly-created temp dir.
+    case "${CHROOT_DIR:?mktemp -d returned empty}" in
+        /tmp/*|/var/tmp/*|"${TMPDIR:-/nonexistent}"/*) ;;
+        *) echo "ERROR: refusing to use unexpected CHROOT_DIR: $CHROOT_DIR" >&2; exit 1 ;;
+    esac
+    # Outputs are written here first, beside $BUILD_DIR so publishing is a rename.
+    STAGE_DIR=$(mktemp -d "$BUILD_DIR/.stage.XXXXXX")
+    CONTAINER_TOOL=""
+    CONTAINER_CID=""
+    trap microvm_cleanup EXIT
+}
+
+microvm_cleanup() {
+        echo "$TAG cleaning up chroot: $CHROOT_DIR"
+        rm -rf "$CHROOT_DIR"
+        rm -rf "${STAGE_DIR:?}"
+        if [ -n "$CONTAINER_CID" ] && [ -n "$CONTAINER_TOOL" ]; then
+            echo "$TAG cleaning up container: $CONTAINER_CID"
+            $CONTAINER_TOOL rm -f "$CONTAINER_CID" >/dev/null 2>&1 || true
+        fi
+}
+
+# microvm_make_rootfs populates $CHROOT_DIR from ALPINE_PACKAGES.
+microvm_make_rootfs() {
+    # --- Populate rootfs ---
+    # Strategy 1: native apk (Alpine hosts, CI)
+    if command -v apk >/dev/null 2>&1; then
+        echo "$TAG rootfs: native apk"
+        mkdir -p \
+            "$CHROOT_DIR/etc/apk" \
+            "$CHROOT_DIR/lib/apk/db" \
+            "$CHROOT_DIR/var/cache/apk"
+        cat > "$CHROOT_DIR/etc/apk/repositories" <<EOF
+${ALPINE_MIRROR}/v${ALPINE_VERSION}/main
+${ALPINE_MIRROR}/v${ALPINE_VERSION}/community
+EOF
+        apk add \
+            --root "$CHROOT_DIR" \
+            --arch x86_64 \
+            --no-cache \
+            --allow-untrusted \
+            --initdb \
+            $ALPINE_PACKAGES
+
+    # Strategy 2: docker -> BuildKit local export.
+    # Do NOT use `docker export` here. On Docker's overlay2 driver it leaks the
+    # container's overlay mount (the mount survives `docker rm`), orphaning layers
+    # under /var/lib/docker that only a daemon restart reclaims; on a busy CI host
+    # this grows without bound. `buildx --output type=local` writes the image
+    # filesystem straight to a directory and releases every mount, so nothing leaks.
+    elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
+        echo "$TAG rootfs: docker buildx --output type=local (alpine:${ALPINE_VERSION})"
+        BUILD_CTX=$(mktemp -d)
+        cat > "$BUILD_CTX/Dockerfile" <<DOCKERFILE
+FROM alpine:${ALPINE_VERSION}
+RUN apk add --no-cache ${ALPINE_PACKAGES}
+DOCKERFILE
+        docker buildx build \
+            --platform linux/amd64 \
+            --output "type=local,dest=${CHROOT_DIR}" \
+            "$BUILD_CTX"
+        rm -rf "$BUILD_CTX"
+
+    # Strategy 3: podman -> run + export. Podman's store does not exhibit the docker
+    # overlay2 export-mount leak, and podman has no buildx.
+    elif command -v podman >/dev/null 2>&1; then
+        echo "$TAG rootfs: podman export (alpine:${ALPINE_VERSION})"
+        CONTAINER_TOOL=podman
+        CONTAINER_CID=$(podman run -d \
+            "alpine:${ALPINE_VERSION}" \
+            sh -c "apk add --no-cache ${ALPINE_PACKAGES}")
+
+        exit_code=$(podman wait "$CONTAINER_CID")
+        if [ "$exit_code" != "0" ]; then
+            echo "ERROR: package installation failed in container (exit $exit_code)" >&2
+            exit 1
+        fi
+
+        podman export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR"
+
+    else
+        echo "ERROR: no rootfs build tool available." >&2
+        echo "       Install one of: apk (Alpine), docker, podman" >&2
+        exit 1
+    fi
+}
+
+microvm_empty_dev() {
+    # --- /dev: empty mountpoint only ---
+    # tar extraction without root silently creates 0-byte regular files for device
+    # nodes. Drop them and leave an empty /dev mountpoint. The two static nodes the
+    # kernel needs (/dev/console to wire init's stdio, /dev/null) are written into
+    # the cpio archive under fakeroot at pack time — NOT created on the host. init
+    # mounts devtmpfs over /dev as its first action for everything else at runtime.
+    rm -rf "${CHROOT_DIR:?}/dev"
+    mkdir "$CHROOT_DIR/dev"
+}
+
+# microvm_locate_kernel finds the kernel and asserts fw_cfg is loadable or built in.
+microvm_locate_kernel() {
+    # --- Locate kernel ---
+    KERNEL_IMG=$(find "$CHROOT_DIR/boot" -name "vmlinuz*" | sort | tail -1)
+    if [ -z "$KERNEL_IMG" ]; then
+        echo "ERROR: no vmlinuz found in $CHROOT_DIR/boot" >&2
+        exit 1
+    fi
+    echo "$TAG kernel image: $KERNEL_IMG"
+
+    KERNEL_VER=$(basename "$KERNEL_IMG" | sed 's/vmlinuz-//')
+    KERNEL_CONFIG="$CHROOT_DIR/boot/config-${KERNEL_VER}"
+
+    # --- Assert fw_cfg module ---
+    if ! find "$CHROOT_DIR/lib/modules" \
+            -name "qemu_fw_cfg.ko*" \
+            -o -name "fw_cfg_sysfs.ko*" \
+        | grep -q .; then
+        if ! grep -q "CONFIG_FW_CFG_SYSFS=y" "$KERNEL_CONFIG" 2>/dev/null; then
+            echo "ERROR: qemu_fw_cfg module missing from initramfs — fw_cfg reads will fail at boot" >&2
+            exit 1
+        fi
+        echo "$TAG fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
+    else
+        echo "$TAG fw_cfg: module found in lib/modules"
+    fi
+}
+
+# --- Strip kernel modules (Phase 2 keep-list approach) ---
+# Only modules needed by a virtio-mmio microvm guest are kept:
+#   virtio*.ko*    — virtio bus, net, blk, rng, console, and helpers
+#   *fw_cfg*.ko*   — QEMU fw_cfg sysfs driver (delivers boot blobs)
+#   $MICROVM_EXTRA_KEEP_GLOBS — caller-named module file globs
+#   $MICROVM_EXTRA_MODULES — caller-named modules, plus dependency closure
+# All other loadable modules are removed. Drivers compiled into the kernel
+# (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
+
+# module_path prints a module's kernel/ path from the stock modules.dep, by
+# name or modules.alias; it fails when the module is not loadable.
+module_path() {
+    local kdir="$1" name="${2//-/_}" path alias_target
+    path=$(awk -F: -v n="$name" '{ f = $1; sub(/.*\//, "", f); sub(/\.ko(\.gz|\.xz|\.zst)?$/, "", f);
+        gsub(/-/, "_", f); if (f == n) { print $1; exit } }' "$kdir/modules.dep")
+    if [ -z "$path" ] && [ -f "$kdir/modules.alias" ]; then
+        alias_target=$(awk -v n="$2" '$1 == "alias" && $2 == n { print $3; exit }' "$kdir/modules.alias")
+        [ -n "$alias_target" ] && [ "$alias_target" != "$2" ] && path=$(module_path "$kdir" "$alias_target")
+    fi
+    [ -n "$path" ] && printf '%s\n' "$path"
+}
+
+# module_builtin reports whether a module name is compiled into the kernel.
+module_builtin() {
+    local kdir="$1" name="${2//-/_}"
+    [ -f "$kdir/modules.builtin" ] || return 1
+    awk -v n="$name" '{ f = $0; sub(/.*\//, "", f); sub(/\.ko$/, "", f); gsub(/-/, "_", f);
+        if (f == n) { found = 1; exit } } END { exit !found }' "$kdir/modules.builtin"
+}
+
+# image_kmod runs the image's own kmod (depmod, modprobe) through its musl
+# loader, so module metadata matches the guest's modprobe without a host kmod.
+image_kmod() {
+    "$CHROOT_DIR/lib/ld-musl-x86_64.so.1" --library-path "$CHROOT_DIR/lib:$CHROOT_DIR/usr/lib" \
+        --argv0 "$1" "$CHROOT_DIR/bin/kmod" "${@:2}"
+}
+
+microvm_strip_modules() {
+    echo "$TAG stripping kernel modules to virtio+fw_cfg+${MICROVM_EXTRA_MODULES} only..."
+    for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
+        kver="$(basename "$kver_dir")"
+        kernel_dir="$kver_dir/kernel"
+        [ -d "$kernel_dir" ] || continue
+
+        if [ ! -s "$kver_dir/modules.dep" ]; then
+            echo "ERROR: stock ${kver_dir}modules.dep missing — cannot resolve module dependencies" >&2
+            exit 1
+        fi
+
+        # Extra modules start the closure; the glob families are kept as found.
+        keep_list=$(mktemp)
+        : > "$keep_list"
+        for name in $MICROVM_EXTRA_MODULES; do
+            if ! path=$(module_path "$kver_dir" "$name"); then
+                if module_builtin "$kver_dir" "$name"; then continue; fi
+                echo "ERROR: module $name is neither loadable nor built into $kver" >&2
+                exit 1
+            fi
+            echo "$path" >> "$keep_list"
+        done
+
+        # Close over modules.dep hard dependencies and modules.softdep pre:/post:
+        # names until the set stops growing. A softdep must be loadable or built in.
+        while :; do
+            before=$(sort -u "$keep_list" | wc -l)
+            for path in $(sort -u "$keep_list"); do
+                awk -F': ' -v p="$path" '$1 == p { n = split($2, d, " "); for (i = 1; i <= n; i++) print d[i] }' \
+                    "$kver_dir/modules.dep" >> "$keep_list"
+                mod=$(basename "$path"); mod="${mod%%.ko*}"; mod="${mod//-/_}"
+                [ -f "$kver_dir/modules.softdep" ] || continue
+                for soft in $(awk -v n="$mod" '{ m = $2; gsub(/-/, "_", m) }
+                        $1 == "softdep" && m == n { for (i = 3; i <= NF; i++) if ($i != "pre:" && $i != "post:") print $i }' \
+                        "$kver_dir/modules.softdep"); do
+                    if soft_path=$(module_path "$kver_dir" "$soft"); then
+                        echo "$soft_path" >> "$keep_list"
+                    elif ! module_builtin "$kver_dir" "$soft"; then
+                        echo "ERROR: softdep $soft of $mod is neither loadable nor built into $kver" >&2
+                        exit 1
+                    fi
+                done
+            done
+            after=$(sort -u "$keep_list" | wc -l)
+            [ "$before" -eq "$after" ] && break
+        done
+
+        extra_names=$(sort -u "$keep_list" | while read -r p; do m=$(basename "$p"); m="${m%%.ko*}"; echo "${m//-/_}"; done)
+        find_args=(-name "virtio*.ko*" -o -name "*fw_cfg*.ko*")
+        for glob in ${MICROVM_EXTRA_KEEP_GLOBS[@]+"${MICROVM_EXTRA_KEEP_GLOBS[@]}"}; do
+            find_args+=(-o -name "$glob")
+        done
+        (cd "$kver_dir" && find kernel "${find_args[@]}") >> "$keep_list"
+
+        # Copy the keeper set out, preserving each kernel/ relative path.
+        tmp_keep=$(mktemp -d)
+        while read -r rel; do
+            rel="${rel#kernel/}"
+            mkdir -p "$tmp_keep/$(dirname "$rel")"
+            cp "$kernel_dir/$rel" "$tmp_keep/$rel"
+        done < <(sort -u "$keep_list")
+        rm -f "$keep_list"
+
+        # Replace kernel module tree with the keeper set.
+        rm -rf "$kernel_dir"
+        mkdir -p "$kernel_dir"
+        # Restore each kept file into the same relative path.
+        (cd "$tmp_keep" && find . -name "*.ko*" | while read -r f; do
+            dest_dir="$kernel_dir/$(dirname "${f#./}")"
+            mkdir -p "$dest_dir"
+            cp "$f" "$dest_dir/"
+        done)
+        rm -rf "$tmp_keep"
+
+        # Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
+        # kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
+        # produces an EMPTY modules.dep from compressed modules — the guest's
+        # modprobe then no-ops and qemu_fw_cfg never loads. Plain .ko sidesteps the
+        # host-kmod dependency entirely and the guest needs no decompressor either.
+        find "$kernel_dir" -name "*.ko.gz" -exec gunzip -f {} +
+
+        # Independent closure gate: each extra-closure module's own .modinfo depends=
+        # must name a kept module or a built-in, so a resolver bug fails here.
+        kept_names=$(find "$kernel_dir" -name "*.ko" -exec basename {} .ko \; | tr '-' '_' | sort -u)
+        while read -r ko; do
+            ko_name=$(basename "$ko" .ko); ko_name="${ko_name//-/_}"
+            grep -qx "$ko_name" <<< "$extra_names" || continue
+            deps=$(tr '\0' '\n' < "$ko" | grep -a -m1 '^depends=' | cut -d= -f2 | tr ',' ' ' || true)
+            for dep in $deps; do
+                dep="${dep//-/_}"
+                if ! grep -qx "$dep" <<< "$kept_names" && ! module_builtin "$kver_dir" "$dep"; then
+                    echo "ERROR: $(basename "$ko") depends on $dep, which is neither kept nor built in" >&2
+                    exit 1
+                fi
+            done
+        done < <(find "$kernel_dir" -name "*.ko")
+        for name in $MICROVM_EXTRA_MODULES; do
+            if ! grep -qx "${name//-/_}" <<< "$kept_names" && ! module_builtin "$kver_dir" "$name"; then
+                echo "ERROR: required module $name missing from the stripped module tree" >&2
+                exit 1
+            fi
+        done
+        echo "$TAG extra module closure: $(echo "$extra_names" | tr '\n' ' ')"
+        echo "$TAG modules kept: $(echo "$kept_names" | tr '\n' ' ')"
+
+        # Rebuild module metadata with the image's own kmod, the binary the guest's
+        # modprobe runs, so no host depmod is needed. Stock indexes still name .ko.gz
+        # files, so they are removed first and a failed rebuild cannot ship them.
+        for f in dep dep.bin alias alias.bin softdep weakdep symbols symbols.bin \
+            builtin.bin builtin.alias.bin devname; do
+            rm -f "$kver_dir/modules.$f"
+        done
+        if ! image_kmod depmod -b "$CHROOT_DIR" "$kver"; then
+            echo "ERROR: image depmod failed for $kver" >&2
+            exit 1
+        fi
+        if [ ! -s "$kver_dir/modules.dep" ] || [ ! -s "$kver_dir/modules.dep.bin" ]; then
+            echo "ERROR: ${kver_dir}modules.dep or modules.dep.bin missing or empty after depmod" >&2
+            exit 1
+        fi
+        # Resolution gate: the image's modprobe must resolve every required module
+        # to plain .ko files present in the stripped tree.
+        for name in $MICROVM_EXTRA_MODULES $MICROVM_GATE_MODULES; do
+            module_builtin "$kver_dir" "$name" && continue
+            if ! plan=$(image_kmod modprobe -d "$CHROOT_DIR" -S "$kver" --show-depends "$name"); then
+                echo "ERROR: image modprobe cannot resolve $name for $kver" >&2
+                exit 1
+            fi
+            while read -r verb path _; do
+                [ "$verb" = "insmod" ] || continue
+                case "$path" in
+                    "$CHROOT_DIR/lib/modules/$kver/kernel/"*.ko) [ -f "$path" ] && continue ;;
+                esac
+                echo "ERROR: modprobe plan for $name names $path, not a kept plain .ko" >&2
+                exit 1
+            done <<< "$plan"
+            echo "$TAG modprobe $name: $(awk '{ n = split($2, a, "/"); printf "%s ", a[n] }' <<< "$plan")"
+        done
+    done
+}
+
+# microvm_strip_rootfs drops package-manager artifacts and debug symbols.
+microvm_strip_rootfs() {
+    # --- Strip package-manager artifacts (not needed at runtime) ---
+    echo "$TAG stripping package manager artifacts..."
+    rm -rf \
+        "$CHROOT_DIR/lib/apk" \
+        "$CHROOT_DIR/var/cache/apk" \
+        "$CHROOT_DIR/etc/apk" \
+        "$CHROOT_DIR/usr/share/man" \
+        "$CHROOT_DIR/usr/share/doc" \
+        "$CHROOT_DIR/usr/share/apk" \
+        "$CHROOT_DIR/usr/include" \
+        "$CHROOT_DIR/usr/lib/pkgconfig"
+
+    # --- Strip debug symbols from binaries ---
+    echo "$TAG stripping debug symbols from binaries..."
+    find "$CHROOT_DIR/usr/sbin" "$CHROOT_DIR/usr/bin" \
+         "$CHROOT_DIR/sbin" "$CHROOT_DIR/bin" \
+        -type f 2>/dev/null | while read -r bin; do
+        strip --strip-all "$bin" 2>/dev/null || true
+    done
+}
+
+# microvm_stage_kernel copies vmlinuz to $STAGE_DIR, then drops /boot from the chroot.
+microvm_stage_kernel() {
+    # --- Copy vmlinuz out before stripping /boot from chroot ---
+    VMLINUZ_OUT="$STAGE_DIR/vmlinuz"
+    echo "$TAG copying kernel: $VMLINUZ_OUT"
+    cp "$KERNEL_IMG" "$VMLINUZ_OUT"
+
+    # Drop /boot from the chroot — vmlinuz, Alpine's stock initramfs-virt, System.map
+    # and kernel config all live here (~25 MiB). None are needed at runtime: the
+    # guest already has the kernel loaded by QEMU's -kernel, and Alpine's initramfs
+    # is unused because we replace it with our own /init. Leaving /boot in the cpio
+    # nearly doubles uncompressed initramfs size and forces a 512 MiB guest just to
+    # survive decompression.
+    rm -rf "${CHROOT_DIR:?}/boot"
+}
+
+microvm_pack_initramfs() {
+    # --- Build initramfs ---
+    INITRAMFS_OUT="$STAGE_DIR/initramfs.cpio.gz"
+    echo "$TAG building initramfs: $INITRAMFS_OUT"
+    # Create the static device nodes and pack the archive inside a single fakeroot
+    # session: mknod/cpio see faked device entries and write real char-device
+    # records into the cpio, while the host filesystem never gets an actual node.
+    fakeroot sh -c '
+        cd "$1" || exit 1
+        mknod -m 600 dev/console c 5 1
+        mknod -m 666 dev/null    c 1 3
+        find . | cpio --quiet -o -H newc | gzip -9 > "$2"
+    ' _ "$CHROOT_DIR" "$INITRAMFS_OUT"
+}

scripts/setup.sh

@@ -732,7 +732,8 @@ table inet spinifex_filter {
         # inventory and running instances to anyone who asks. 8660 is
         # predastore's unauthenticated /healthz and /readyz, which name the
         # peers a gate cannot reach, so it belongs here for the same reason.
-        ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
+        # 5001 is the admin network test, listening only while one runs.
+        ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5001, 5300, 6641, 6642, 6643, 6644, 8660 } accept
         ip saddr $spinifex_peers udp dport { 5300, 6660, 7660 } accept
 
         # Encap plane, peer-scoped: Geneve, IKE, NAT-T and ESP. Geneve is a
@@ -1309,6 +1310,7 @@ qemu-utils gdisk ovmf qemu-efi-aarch64 less
 libvirt-daemon-system libvirt-clients
 pciutils
 jq curl iproute2 iputils-arping ethtool netcat-openbsd wget unzip xz-utils file
+dmidecode iperf
 ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base
 chrony nftables"
 
@@ -1318,7 +1320,7 @@ chrony nftables"
 OL_RUNTIME_PACKAGES="nbdkit nbdkit-basic-filters
 qemu-kvm qemu-img edk2-ovmf seabios-bin less
 pciutils
-jq curl iproute iputils ethtool nmap-ncat wget unzip xz file
+jq curl iproute iputils ethtool nmap-ncat wget unzip xz file dmidecode
 ovn25.03-central ovn25.03-host openvswitch3.5 openvswitch3.5-ipsec libreswan dhcpcd
 chrony nftables iptables-nft systemd-resolved"
 

scripts/terraform/vultr-spx/deploy-vultr.sh

@@ -329,6 +329,18 @@ restart_spinifex() {
     die "$NODE_NAME never logged its Vultr allocator; see journalctl -u spinifex-daemon on $IP"
 }
 
+# public_default_subnets turns on auto-assign for the default subnets formation
+# created before the Vultr pool existed, as AWS default subnets have it. Later
+# accounts are seeded with it by the daemon, which now sees the pool.
+public_default_subnets() {
+    log "auto-assigning public addresses in the default subnets"
+    on 'export AWS_PROFILE=spinifex
+        for s in $(aws ec2 describe-subnets --filters Name=default-for-az,Values=true \
+                --query "Subnets[?MapPublicIpOnLaunch==\`false\`].SubnetId" --output text); do
+            aws ec2 modify-subnet-attribute --subnet-id "$s" --map-public-ip-on-launch || exit 1
+        done' || die "could not turn on public addresses in the default subnets on $IP"
+}
+
 import_image() {
     log "importing $IMAGE"
     on "spx admin images import --name $IMAGE --config /etc/spinifex/spinifex.toml >/root/image-import.log 2>&1" ||
@@ -368,6 +380,7 @@ cmd_up() {
     for_nodes pool configure_pool
     for_nodes restart restart_spinifex
     IP="${IPS[0]}"
+    done_step public || { public_default_subnets; mark_step public; }
     if [ -n "$IMAGE" ]; then done_step image || { import_image; mark_step image; }; fi
     write_profile
     log "ready: https://$IP:9999 (profile $PROFILE)"

spinifex/agents/lambda/agent/agent.go

@@ -0,0 +1,91 @@
+// Package agent runs the guest loop: take a request from the host link, hand
+// it to the function runtime, and report the outcome back.
+package agent
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+)
+
+const (
+	errTypeTimedOut = "Sandbox.Timedout"
+	errTypeUnknown  = "Runtime.Unknown"
+)
+
+// ErrInvocationTimedOut stops the agent after it reports a timed-out
+// invocation, because the runtime may still be running it.
+var ErrInvocationTimedOut = errors.New("invocation timed out; environment must not serve another request")
+
+// Run serves requests from link until the host asks for shutdown (nil), the
+// runtime reports an init error, an invocation times out, ctx ends, or the
+// link fails. GrantID and AttemptID are echoed to the host unchanged.
+func Run(ctx context.Context, link hostlink.Link, srv *runtimeapi.Server) error {
+	ctx, cancel := context.WithCancelCause(ctx)
+	defer cancel(nil)
+	go func() {
+		select {
+		case <-srv.InitFailed():
+			ie, _ := srv.InitError()
+			cancel(fmt.Errorf("runtime init error %q: %s", ie.Type, ie.Body))
+		case <-ctx.Done():
+		}
+	}()
+
+	for {
+		req, err := link.Next(ctx)
+		if errors.Is(err, hostlink.ErrShutdown) {
+			return nil
+		}
+		if err != nil {
+			return cause(ctx, err)
+		}
+
+		res, err := srv.Invoke(ctx, runtimeapi.Invocation{
+			RequestID:   req.RequestID,
+			FunctionARN: req.FunctionARN,
+			Deadline:    time.UnixMilli(req.DeadlineMs),
+			TraceID:     req.TraceID,
+			Payload:     req.Payload,
+		})
+		timedOut := errors.Is(err, runtimeapi.ErrDeadlineExceeded)
+		if err != nil {
+			if ctx.Err() != nil {
+				return cause(ctx, err)
+			}
+			res = failure(err)
+		}
+		resp := hostlink.Response{
+			GrantID:       req.GrantID,
+			AttemptID:     req.AttemptID,
+			RequestID:     req.RequestID,
+			Payload:       res.Payload,
+			FunctionError: res.FunctionError,
+		}
+		if err := link.Complete(ctx, resp); err != nil {
+			return cause(ctx, err)
+		}
+		if timedOut {
+			return ErrInvocationTimedOut
+		}
+	}
+}
+
+func cause(ctx context.Context, err error) error {
+	if ctx.Err() != nil {
+		return context.Cause(ctx)
+	}
+	return err
+}
+
+func failure(err error) runtimeapi.Result {
+	errType := errTypeUnknown
+	if errors.Is(err, runtimeapi.ErrDeadlineExceeded) {
+		errType = errTypeTimedOut
+	}
+	return runtimeapi.Result{Payload: runtimeapi.ErrorBody(err.Error(), errType), FunctionError: errType}
+}

spinifex/agents/lambda/agent/agent_test.go

@@ -0,0 +1,142 @@
+package agent
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+)
+
+const api = "/2018-06-01/runtime/"
+
+// fakeLink feeds scripted requests and records completions.
+type fakeLink struct {
+	reqs []hostlink.Request
+	got  chan hostlink.Response
+}
+
+var _ hostlink.Link = (*fakeLink)(nil)
+
+func (l *fakeLink) Next(ctx context.Context) (hostlink.Request, error) {
+	if len(l.reqs) == 0 {
+		return hostlink.Request{}, hostlink.ErrShutdown
+	}
+	r := l.reqs[0]
+	l.reqs = l.reqs[1:]
+	return r, nil
+}
+
+func (l *fakeLink) Complete(ctx context.Context, r hostlink.Response) error {
+	l.got <- r
+	return nil
+}
+
+// runtime polls next and answers using reply.
+func runtime(t *testing.T, base string, reply func(id string, body []byte) (path string, out string)) {
+	t.Helper()
+	go func() {
+		for {
+			resp, err := http.Get(base + api + "invocation/next")
+			if err != nil {
+				return
+			}
+			body, _ := io.ReadAll(resp.Body)
+			resp.Body.Close()
+			id := resp.Header.Get("Lambda-Runtime-Aws-Request-Id")
+			path, out := reply(id, body)
+			r, err := http.Post(base+api+"invocation/"+id+path, "application/json", strings.NewReader(out))
+			if err != nil {
+				return
+			}
+			r.Body.Close()
+		}
+	}()
+}
+
+func deadline(d time.Duration) int64 { return time.Now().Add(d).UnixMilli() }
+
+func TestRunServesRequestsAndEchoesIDs(t *testing.T) {
+	srv := runtimeapi.New()
+	ts := httptest.NewServer(srv.Handler())
+	defer ts.Close()
+	runtime(t, ts.URL, func(id string, body []byte) (string, string) {
+		if string(body) == "bad" {
+			return "/error", `{"errorMessage":"no"}`
+		}
+		return "/response", "echo:" + string(body)
+	})
+
+	link := &fakeLink{
+		got: make(chan hostlink.Response, 4),
+		reqs: []hostlink.Request{
+			{GrantID: "g1", AttemptID: "a1", RequestID: "r1", FunctionARN: "arn", DeadlineMs: deadline(2 * time.Second), Payload: []byte("one")},
+			{GrantID: "g2", AttemptID: "a2", RequestID: "r2", FunctionARN: "arn", DeadlineMs: deadline(2 * time.Second), Payload: []byte("bad")},
+		},
+	}
+	require.NoError(t, Run(context.Background(), link, srv))
+
+	r1, r2 := <-link.got, <-link.got
+	assert.Equal(t, hostlink.Response{GrantID: "g1", AttemptID: "a1", RequestID: "r1", Payload: []byte("echo:one")}, r1)
+	assert.Equal(t, "g2", r2.GrantID)
+	assert.Equal(t, "a2", r2.AttemptID)
+	assert.Equal(t, "Unhandled", r2.FunctionError)
+}
+
+func TestRunReportsTimeoutAndStops(t *testing.T) {
+	srv := runtimeapi.New() // no runtime polling: the invocation expires
+	link := &fakeLink{
+		got: make(chan hostlink.Response, 2),
+		reqs: []hostlink.Request{
+			{GrantID: "g", AttemptID: "a", RequestID: "r", DeadlineMs: deadline(30 * time.Millisecond)},
+			{GrantID: "g2", AttemptID: "a2", RequestID: "r2", DeadlineMs: deadline(2 * time.Second)},
+		},
+	}
+	require.ErrorIs(t, Run(context.Background(), link, srv), ErrInvocationTimedOut)
+	r := <-link.got
+	assert.Equal(t, "Sandbox.Timedout", r.FunctionError)
+	assert.Equal(t, "g", r.GrantID)
+	assert.Len(t, link.reqs, 1, "the next request must not be taken")
+	assert.Empty(t, link.got, "nothing else is reported")
+}
+
+// blockLink blocks in Next until ctx ends.
+type blockLink struct{ fakeLink }
+
+func (blockLink) Next(ctx context.Context) (hostlink.Request, error) {
+	<-ctx.Done()
+	return hostlink.Request{}, ctx.Err()
+}
+
+func TestRunStopsOnInitError(t *testing.T) {
+	srv := runtimeapi.New()
+	ts := httptest.NewServer(srv.Handler())
+	defer ts.Close()
+	r, err := http.Post(ts.URL+api+"init/error", "application/json", strings.NewReader(`{"errorMessage":"x"}`))
+	require.NoError(t, err)
+	r.Body.Close()
+
+	err = Run(context.Background(), &blockLink{}, srv)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "init error")
+}
+
+type errLink struct{ fakeLink }
+
+func (errLink) Next(context.Context) (hostlink.Request, error) {
+	return hostlink.Request{}, errors.New("link down")
+}
+
+func TestRunReturnsLinkError(t *testing.T) {
+	err := Run(context.Background(), &errLink{}, runtimeapi.New())
+	require.EqualError(t, err, "link down")
+}

spinifex/agents/lambda/agent/bootstrap.go

@@ -0,0 +1,57 @@
+//go:build linux
+
+package agent
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os/exec"
+	"path/filepath"
+	goruntime "runtime"
+	"syscall"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// sandboxID is the uid and gid the function's bootstrap runs as.
+const sandboxID = 993
+
+// StartBootstrap starts the function's bootstrap at path as the sandbox uid and
+// gid with no supplementary groups, with no_new_privs, and with exactly env,
+// in the bootstrap's own directory. The caller waits on the returned command.
+func StartBootstrap(ctx context.Context, path string, env []string, stdout, stderr io.Writer) (*exec.Cmd, error) {
+	cred := &syscall.Credential{Uid: sandboxID, Gid: sandboxID, Groups: []uint32{}}
+	return startBootstrap(ctx, path, env, stdout, stderr, cred)
+}
+
+func startBootstrap(ctx context.Context, path string, env []string, stdout, stderr io.Writer, cred *syscall.Credential) (*exec.Cmd, error) {
+	cmd := exec.CommandContext(ctx, path)
+	// A nil Env inherits the agent's environment; the bootstrap gets only env.
+	cmd.Env = append([]string{}, env...)
+	cmd.Stdout, cmd.Stderr = stdout, stderr
+	// Packaged functions open their files relative to the task root.
+	cmd.Dir = filepath.Dir(path)
+	if cred != nil {
+		cmd.SysProcAttr = &syscall.SysProcAttr{Credential: cred}
+	}
+	cmd.Cancel = func() error { return cmd.Process.Signal(syscall.SIGTERM) }
+	cmd.WaitDelay = 5 * time.Second
+
+	started := make(chan error, 1)
+	go func() {
+		// no_new_privs is per thread and the child inherits it from this one. The
+		// thread is never unlocked, so Go ends it when this goroutine returns.
+		goruntime.LockOSThread()
+		if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {
+			started <- fmt.Errorf("set no_new_privs: %w", err)
+			return
+		}
+		started <- cmd.Start()
+	}()
+	if err := <-started; err != nil {
+		return nil, err
+	}
+	return cmd, nil
+}

spinifex/agents/lambda/agent/bootstrap_test.go

@@ -0,0 +1,179 @@
+//go:build linux
+
+package agent
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	goruntime "runtime"
+	"strings"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// statusProbe writes an executable that prints its own credential lines.
+func statusProbe(t *testing.T) string {
+	t.Helper()
+	// The sandbox uid must be able to reach the probe through the test's dirs.
+	dir := t.TempDir()
+	require.NoError(t, os.Chmod(filepath.Dir(dir), 0o755))
+	require.NoError(t, os.Chmod(dir, 0o755))
+	path := filepath.Join(dir, "bootstrap")
+	script := "#!/bin/sh\ngrep -E '^(Uid|Gid|Groups|NoNewPrivs):' /proc/self/status\n"
+	require.NoError(t, os.WriteFile(path, []byte(script), 0o755))
+	return path
+}
+
+func statusLines(t *testing.T, out []byte) map[string]string {
+	t.Helper()
+	lines := map[string]string{}
+	for l := range strings.SplitSeq(strings.TrimSpace(string(out)), "\n") {
+		k, v, ok := strings.Cut(l, ":")
+		require.True(t, ok, "status line %q", l)
+		lines[k] = strings.Join(strings.Fields(v), " ")
+	}
+	return lines
+}
+
+// run starts a bootstrap with the given credential and returns its stdout.
+func run(t *testing.T, path string, env []string, sandbox bool) []byte {
+	t.Helper()
+	var out bytes.Buffer
+	start := startBootstrap
+	if sandbox {
+		start = func(ctx context.Context, p string, e []string, o, s io.Writer, _ *syscall.Credential) (*exec.Cmd, error) {
+			return StartBootstrap(ctx, p, e, o, s)
+		}
+	}
+	cmd, err := start(context.Background(), path, env, &out, os.Stderr, nil)
+	require.NoError(t, err)
+	require.NoError(t, cmd.Wait())
+	return out.Bytes()
+}
+
+// threadNoNewPrivs reads no_new_privs for the calling OS thread.
+func threadNoNewPrivs(t *testing.T) string {
+	t.Helper()
+	b, err := os.ReadFile("/proc/thread-self/status")
+	require.NoError(t, err)
+	return statusLines(t, grepLines(b, "NoNewPrivs:"))["NoNewPrivs"]
+}
+
+func grepLines(b []byte, prefix string) []byte {
+	var keep []string
+	for l := range strings.SplitSeq(string(b), "\n") {
+		if strings.HasPrefix(l, prefix) {
+			keep = append(keep, l)
+		}
+	}
+	return []byte(strings.Join(keep, "\n"))
+}
+
+func TestStartBootstrapSetsNoNewPrivsWithoutChangingTheCaller(t *testing.T) {
+	goruntime.LockOSThread()
+	defer goruntime.UnlockOSThread()
+	if threadNoNewPrivs(t) != "0" {
+		t.Skip("test thread already has no_new_privs; the child would inherit it")
+	}
+
+	out := run(t, statusProbe(t), nil, false)
+	assert.Equal(t, "1", statusLines(t, out)["NoNewPrivs"])
+	assert.Equal(t, "0", threadNoNewPrivs(t), "the calling thread must keep no_new_privs unset")
+}
+
+func TestStartBootstrapDropsToSandboxIdentity(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip("dropping to the sandbox uid needs root")
+	}
+	lines := statusLines(t, run(t, statusProbe(t), nil, true))
+	assert.Equal(t, "993 993 993 993", lines["Uid"])
+	assert.Equal(t, "993 993 993 993", lines["Gid"])
+	assert.Empty(t, lines["Groups"])
+	assert.Equal(t, "1", lines["NoNewPrivs"])
+}
+
+// A tenant LD_PRELOAD constructor runs before the bootstrap's main; it must
+// already see no_new_privs, so no loader runs before the flag is set.
+func TestStartBootstrapPreloadConstructorSeesNoNewPrivs(t *testing.T) {
+	goruntime.LockOSThread()
+	defer goruntime.UnlockOSThread()
+	if threadNoNewPrivs(t) != "0" {
+		t.Skip("test thread already has no_new_privs; the check would pass vacuously")
+	}
+	cc, err := exec.LookPath("cc")
+	if err != nil {
+		t.Skip("no C compiler for the preload library")
+	}
+	bootstrap, err := exec.LookPath("true")
+	require.NoError(t, err)
+	dir := t.TempDir()
+	src := filepath.Join(dir, "preload.c")
+	lib := filepath.Join(dir, "preload.so")
+	require.NoError(t, os.WriteFile(src, []byte(preloadSource), 0o644))
+	build := exec.Command(cc, "-shared", "-fPIC", "-o", lib, src)
+	build.Stderr = os.Stderr
+	require.NoError(t, build.Run())
+
+	out := run(t, bootstrap, []string{"LD_PRELOAD=" + lib}, false)
+	assert.Equal(t, "preload NoNewPrivs: 1", strings.Join(strings.Fields(string(out)), " "))
+}
+
+const preloadSource = `#include <stdio.h>
+#include <string.h>
+
+__attribute__((constructor)) static void report(void) {
+	char line[256];
+	FILE *f = fopen("/proc/self/status", "r");
+	if (!f)
+		return;
+	while (fgets(line, sizeof line, f))
+		if (strncmp(line, "NoNewPrivs:", 11) == 0)
+			printf("preload %s", line);
+	fclose(f);
+	fflush(stdout);
+}
+`
+
+func TestStartBootstrapDoesNotInheritAgentEnvironment(t *testing.T) {
+	t.Setenv("ZZ_AGENT_SECRET", "must-not-leak")
+	probe, err := exec.LookPath("env")
+	require.NoError(t, err)
+
+	for name, env := range map[string][]string{"nil": nil, "empty": {}, "given": {"ONLY=this"}} {
+		out := run(t, probe, env, false)
+		// Never print the captured environment: an inherited one may hold secrets.
+		if strings.TrimSuffix(string(out), "\n") != strings.Join(env, "\n") {
+			t.Errorf("%s: bootstrap environment is not exactly the one given", name)
+		}
+	}
+}
+
+// Without root the sandbox credential cannot be applied, so the exported entry
+// point must refuse rather than run the bootstrap as the caller.
+func TestStartBootstrapAsNonRootRefusesInsteadOfKeepingCallerIdentity(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("root can apply the sandbox credential")
+	}
+	_, err := StartBootstrap(context.Background(), statusProbe(t), nil, nil, nil)
+	assert.ErrorIs(t, err, syscall.EPERM)
+}
+
+func TestStartBootstrapMissingExecutableFails(t *testing.T) {
+	_, err := startBootstrap(context.Background(), filepath.Join(t.TempDir(), "bootstrap"), nil, nil, nil, nil)
+	assert.Error(t, err)
+}
+
+func TestStartBootstrapRunsInItsOwnDirectory(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "data.txt"), []byte("packaged"), 0o644))
+	bootstrap := filepath.Join(dir, "bootstrap")
+	require.NoError(t, os.WriteFile(bootstrap, []byte("#!/bin/sh\ncat data.txt\n"), 0o755))
+	assert.Equal(t, "packaged", string(run(t, bootstrap, []string{"PATH=/bin:/usr/bin"}, false)))
+}

spinifex/agents/lambda/hostlink/link.go

@@ -0,0 +1,41 @@
+// Package hostlink is the guest's boundary to host control: an authenticated
+// TCP connection under per-generation mutual TLS. The message protocol over it
+// is not yet chosen; implementations of Link are carriers.
+package hostlink
+
+import (
+	"context"
+	"errors"
+)
+
+// ErrShutdown is returned by Link.Next when the host asks the agent to stop.
+var ErrShutdown = errors.New("hostlink: host requested shutdown")
+
+// Request is one invocation from the host. GrantID and AttemptID are opaque
+// to the agent and are echoed unchanged in the Response.
+type Request struct {
+	GrantID     string `json:"grant_id"`
+	AttemptID   string `json:"attempt_id"`
+	RequestID   string `json:"request_id"`
+	FunctionARN string `json:"function_arn"`
+	TraceID     string `json:"trace_id,omitempty"`
+	DeadlineMs  int64  `json:"deadline_ms"`
+	Payload     []byte `json:"payload"`
+}
+
+// Response is the outcome of a Request. FunctionError is empty on success.
+type Response struct {
+	GrantID       string `json:"grant_id"`
+	AttemptID     string `json:"attempt_id"`
+	RequestID     string `json:"request_id"`
+	Payload       []byte `json:"payload"`
+	FunctionError string `json:"function_error,omitempty"`
+}
+
+// Link carries invocations from the host to the guest and results back.
+type Link interface {
+	// Next blocks for the next request, or returns ErrShutdown.
+	Next(ctx context.Context) (Request, error)
+	// Complete reports the outcome of a request.
+	Complete(ctx context.Context, resp Response) error
+}

spinifex/agents/lambda/hostlink/seal.go

@@ -0,0 +1,42 @@
+package hostlink
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+// FwCfgRoot is the sysfs tree the qemu_fw_cfg module exposes; removing the
+// module removes every entry under it, including the guest key.
+const FwCfgRoot = "/sys/firmware/qemu_fw_cfg"
+
+// UnloadFwCfg removes the qemu_fw_cfg module from the running kernel.
+func UnloadFwCfg() error {
+	if err := unix.DeleteModule("qemu_fw_cfg", 0); err != nil {
+		return fmt.Errorf("hostlink: unload qemu_fw_cfg: %w", err)
+	}
+	return nil
+}
+
+// SealFwCfg makes fw_cfg unreadable for the rest of the boot. It succeeds only
+// once root is gone, so a built-in or busy module fails closed rather than
+// leaving the guest key readable by code started later.
+func SealFwCfg(root string, unload func() error) error {
+	if _, err := os.Stat(root); errors.Is(err, fs.ErrNotExist) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("hostlink: fw_cfg: %w", err)
+	}
+	unloadErr := unload()
+	_, err := os.Stat(root)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil
+	}
+	if unloadErr == nil {
+		unloadErr = errors.New("unload reported success")
+	}
+	return fmt.Errorf("hostlink: fw_cfg still readable at %s: %w", root, errors.Join(unloadErr, err))
+}

spinifex/agents/lambda/hostlink/seal_test.go

@@ -0,0 +1,43 @@
+package hostlink
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func fwCfgTree(t *testing.T) string {
+	t.Helper()
+	root := filepath.Join(t.TempDir(), "qemu_fw_cfg")
+	require.NoError(t, os.MkdirAll(filepath.Join(root, "by_name"), 0o700))
+	return root
+}
+
+func TestSealFwCfg(t *testing.T) {
+	t.Run("already absent needs no unload", func(t *testing.T) {
+		called := false
+		require.NoError(t, SealFwCfg(filepath.Join(t.TempDir(), "absent"), func() error { called = true; return nil }))
+		assert.False(t, called)
+	})
+	t.Run("unload removes the tree", func(t *testing.T) {
+		root := fwCfgTree(t)
+		require.NoError(t, SealFwCfg(root, func() error { return os.RemoveAll(root) }))
+	})
+	t.Run("unload fails and the tree remains", func(t *testing.T) {
+		builtIn := errors.New("module is built in")
+		err := SealFwCfg(fwCfgTree(t), func() error { return builtIn })
+		assert.ErrorIs(t, err, builtIn)
+		assert.ErrorContains(t, err, "still readable")
+	})
+	t.Run("unload succeeds but the tree remains", func(t *testing.T) {
+		assert.ErrorContains(t, SealFwCfg(fwCfgTree(t), func() error { return nil }), "unload reported success")
+	})
+	t.Run("unload errors after removing the tree", func(t *testing.T) {
+		root := fwCfgTree(t)
+		require.NoError(t, SealFwCfg(root, func() error { _ = os.RemoveAll(root); return errors.New("late") }))
+	})
+}

spinifex/agents/lambda/hostlink/test_stream.go

@@ -0,0 +1,163 @@
+//go:build lambdatestlink
+
+package hostlink
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"sync"
+)
+
+// maxFrame bounds one frame: a 6 MiB payload as base64 plus headroom.
+const maxFrame = (6<<20+2)/3*4 + 64<<10
+
+const (
+	typeInvoke   = "invoke"
+	typeShutdown = "shutdown"
+	typeResult   = "result"
+)
+
+// ErrFrameTooLarge is returned for a frame beyond the size bound.
+var ErrFrameTooLarge = errors.New("hostlink: frame too large")
+
+type frame struct {
+	Type     string    `json:"type"`
+	Request  *Request  `json:"request,omitempty"`
+	Response *Response `json:"response,omitempty"`
+}
+
+// TestStream is a test-only carrier over an io.ReadWriter, framing JSON with
+// a 4-byte big-endian length. It is not the guest protocol.
+type TestStream struct {
+	rw  io.ReadWriter
+	in  chan frame
+	wmu sync.Mutex
+
+	err error // set before in is closed
+}
+
+var _ Link = (*TestStream)(nil)
+
+// NewTestStream starts reading frames from rw.
+func NewTestStream(rw io.ReadWriter) *TestStream {
+	s := &TestStream{rw: rw, in: make(chan frame)}
+	go s.readLoop()
+	return s
+}
+
+func (s *TestStream) readLoop() {
+	defer close(s.in)
+	for {
+		f, err := readFrame(s.rw)
+		if err != nil {
+			s.err = err
+			return
+		}
+		s.in <- f
+	}
+}
+
+func (s *TestStream) recv(ctx context.Context) (frame, error) {
+	select {
+	case f, ok := <-s.in:
+		if !ok {
+			return frame{}, s.err
+		}
+		return f, nil
+	case <-ctx.Done():
+		return frame{}, ctx.Err()
+	}
+}
+
+// Next implements Link.
+func (s *TestStream) Next(ctx context.Context) (Request, error) {
+	f, err := s.recv(ctx)
+	if err != nil {
+		return Request{}, err
+	}
+	switch f.Type {
+	case typeShutdown:
+		return Request{}, ErrShutdown
+	case typeInvoke:
+		if f.Request == nil {
+			return Request{}, errors.New("hostlink: invoke frame without request")
+		}
+		return *f.Request, nil
+	default:
+		return Request{}, fmt.Errorf("hostlink: unexpected frame type %q", f.Type)
+	}
+}
+
+// Complete implements Link.
+func (s *TestStream) Complete(ctx context.Context, resp Response) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	return s.write(frame{Type: typeResult, Response: &resp})
+}
+
+// HostSend writes an invoke frame, as the host side of the carrier.
+func (s *TestStream) HostSend(req Request) error {
+	return s.write(frame{Type: typeInvoke, Request: &req})
+}
+
+// HostShutdown writes a shutdown frame, as the host side of the carrier.
+func (s *TestStream) HostShutdown() error {
+	return s.write(frame{Type: typeShutdown})
+}
+
+// HostResult reads a result frame, as the host side of the carrier.
+func (s *TestStream) HostResult(ctx context.Context) (Response, error) {
+	f, err := s.recv(ctx)
+	if err != nil {
+		return Response{}, err
+	}
+	if f.Type != typeResult || f.Response == nil {
+		return Response{}, fmt.Errorf("hostlink: unexpected frame type %q", f.Type)
+	}
+	return *f.Response, nil
+}
+
+func (s *TestStream) write(f frame) error {
+	b, err := json.Marshal(f)
+	if err != nil {
+		return err
+	}
+	if len(b) > maxFrame {
+		return ErrFrameTooLarge
+	}
+	buf := make([]byte, 4+len(b))
+	binary.BigEndian.PutUint32(buf, uint32(len(b)))
+	copy(buf[4:], b)
+	s.wmu.Lock()
+	defer s.wmu.Unlock()
+	_, err = s.rw.Write(buf)
+	return err
+}
+
+func readFrame(r io.Reader) (frame, error) {
+	var hdr [4]byte
+	if _, err := io.ReadFull(r, hdr[:]); err != nil {
+		return frame{}, err
+	}
+	n := binary.BigEndian.Uint32(hdr[:])
+	if n > maxFrame {
+		return frame{}, ErrFrameTooLarge
+	}
+	body := make([]byte, n)
+	if _, err := io.ReadFull(r, body); err != nil {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		return frame{}, err
+	}
+	var f frame
+	if err := json.Unmarshal(body, &f); err != nil {
+		return frame{}, fmt.Errorf("hostlink: bad frame: %w", err)
+	}
+	return f, nil
+}

spinifex/agents/lambda/hostlink/test_stream_test.go

@@ -0,0 +1,73 @@
+//go:build lambdatestlink
+
+package hostlink
+
+import (
+	"context"
+	"encoding/binary"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// pair returns the guest carrier and the raw host end of a pipe.
+func pair(t *testing.T) (*TestStream, net.Conn) {
+	t.Helper()
+	g, h := net.Pipe()
+	t.Cleanup(func() { g.Close(); h.Close() })
+	return NewTestStream(g), h
+}
+
+func TestRoundTrip(t *testing.T) {
+	guest, host := pair(t)
+	hostEnd := NewTestStream(host)
+	req := Request{GrantID: "g", AttemptID: "a", RequestID: "r", FunctionARN: "arn", TraceID: "t", DeadlineMs: 42, Payload: []byte{0, 1, 2, 255}}
+	go func() { _ = hostEnd.HostSend(req) }()
+
+	got, err := guest.Next(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, req, got)
+
+	resp := Response{GrantID: "g", AttemptID: "a", RequestID: "r", Payload: []byte("out"), FunctionError: "Unhandled"}
+	go func() { _ = guest.Complete(context.Background(), resp) }()
+	gotResp, err := hostEnd.HostResult(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, resp, gotResp)
+}
+
+func TestOversizeFrameRefused(t *testing.T) {
+	guest, host := pair(t)
+	go func() {
+		var hdr [4]byte
+		binary.BigEndian.PutUint32(hdr[:], maxFrame+1)
+		_, _ = host.Write(hdr[:])
+	}()
+	_, err := guest.Next(context.Background())
+	require.ErrorIs(t, err, ErrFrameTooLarge)
+
+	err = guest.Complete(context.Background(), Response{Payload: make([]byte, maxFrame)})
+	require.ErrorIs(t, err, ErrFrameTooLarge)
+}
+
+func TestTruncatedFrameErrors(t *testing.T) {
+	guest, host := pair(t)
+	go func() {
+		var hdr [4]byte
+		binary.BigEndian.PutUint32(hdr[:], 100)
+		_, _ = host.Write(append(hdr[:], []byte(`{"type"`)...))
+		host.Close()
+	}()
+	_, err := guest.Next(context.Background())
+	require.Error(t, err)
+	assert.NotErrorIs(t, err, ErrShutdown)
+}
+
+func TestShutdownSurfaced(t *testing.T) {
+	guest, host := pair(t)
+	hostEnd := NewTestStream(host)
+	go func() { _ = hostEnd.HostShutdown() }()
+	_, err := guest.Next(context.Background())
+	require.ErrorIs(t, err, ErrShutdown)
+}

spinifex/agents/lambda/hostlink/tls.go

@@ -0,0 +1,115 @@
+package hostlink
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/pki"
+)
+
+// FwCfgDir is where the guest kernel exposes the executor's fw_cfg entries.
+const FwCfgDir = FwCfgRoot + "/by_name/" + lambdav1.FwCfgPrefix
+
+// Identity is the guest's material for one environment generation.
+type Identity struct {
+	Cert     tls.Certificate
+	HostCert *x509.Certificate
+	Scope    lambdav1.Scope
+	HostAddr string
+}
+
+// LoadIdentity reads the executor's fw_cfg entries from dir.
+func LoadIdentity(dir string) (Identity, error) {
+	read := func(name string) ([]byte, error) {
+		b, err := os.ReadFile(filepath.Join(dir, name, "raw"))
+		if err != nil {
+			return nil, fmt.Errorf("hostlink: read %s: %w", name, err)
+		}
+		return b, nil
+	}
+	var id Identity
+	keyPEM, err := read(lambdav1.EntryGuestKey)
+	if err != nil {
+		return id, err
+	}
+	certPEM, err := read(lambdav1.EntryGuestCert)
+	if err != nil {
+		return id, err
+	}
+	hostPEM, err := read(lambdav1.EntryHostCert)
+	if err != nil {
+		return id, err
+	}
+	scope, err := read(lambdav1.EntryScope)
+	if err != nil {
+		return id, err
+	}
+	addr, err := read(lambdav1.EntryHostAddr)
+	if err != nil {
+		return id, err
+	}
+
+	if id.Cert, err = tls.X509KeyPair(certPEM, keyPEM); err != nil {
+		return Identity{}, fmt.Errorf("hostlink: guest key pair: %w", err)
+	}
+	blk, rest := pem.Decode(hostPEM)
+	if blk == nil || blk.Type != "CERTIFICATE" || len(strings.TrimSpace(string(rest))) != 0 {
+		return Identity{}, errors.New("hostlink: host certificate: want exactly one PEM certificate")
+	}
+	if id.HostCert, err = x509.ParseCertificate(blk.Bytes); err != nil {
+		return Identity{}, fmt.Errorf("hostlink: host certificate: %w", err)
+	}
+	if id.Scope, err = lambdav1.ParseScope(strings.TrimSpace(string(scope))); err != nil {
+		return Identity{}, err
+	}
+	id.HostAddr = strings.TrimSpace(string(addr))
+	if _, _, err := net.SplitHostPort(id.HostAddr); err != nil {
+		return Identity{}, fmt.Errorf("hostlink: host address: %w", err)
+	}
+	return id, nil
+}
+
+// ClientConfig is the guest's TLS 1.3 configuration. The host has no DNS name,
+// so VerifyConnection checks the pinned host certificate and its URI SAN.
+func (id Identity) ClientConfig() *tls.Config {
+	return &tls.Config{
+		MinVersion:         tls.VersionTLS13,
+		MaxVersion:         tls.VersionTLS13,
+		Certificates:       []tls.Certificate{id.Cert},
+		NextProtos:         []string{lambdav1.ALPNv1},
+		InsecureSkipVerify: true, //nolint:gosec // VerifyConnection performs full validation against the pinned host certificate.
+		VerifyConnection: func(cs tls.ConnectionState) error {
+			if cs.NegotiatedProtocol != lambdav1.ALPNv1 {
+				return fmt.Errorf("hostlink: application protocol %q, want %q", cs.NegotiatedProtocol, lambdav1.ALPNv1)
+			}
+			return pki.VerifyPinned(cs.PeerCertificates, id.HostCert, x509.ExtKeyUsageServerAuth, id.Scope.URI(lambdav1.RoleHost))
+		},
+	}
+}
+
+// Dial opens the authenticated control connection to the host. Under TLS 1.3
+// the host may still refuse the guest's certificate after Dial returns; that
+// surfaces on the first read, so the caller must not treat Dial as acceptance.
+func Dial(ctx context.Context, id Identity) (*tls.Conn, error) {
+	d := tls.Dialer{NetDialer: &net.Dialer{Timeout: 10 * time.Second}, Config: id.ClientConfig()}
+	c, err := d.DialContext(ctx, "tcp", id.HostAddr)
+	if err != nil {
+		return nil, fmt.Errorf("hostlink: dial host %s: %w", id.HostAddr, err)
+	}
+	tc, ok := c.(*tls.Conn)
+	if !ok {
+		_ = c.Close()
+		return nil, fmt.Errorf("hostlink: dial host %s: unexpected connection type %T", id.HostAddr, c)
+	}
+	return tc, nil
+}

spinifex/agents/lambda/hostlink/tls_test.go

@@ -0,0 +1,291 @@
+package hostlink
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"net/url"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/pki"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	gen1 lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-1"
+	gen2 lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-2"
+)
+
+type testIdent struct {
+	cert    tls.Certificate
+	leaf    *x509.Certificate
+	certPEM []byte
+	keyPEM  []byte
+}
+
+func newTestIdent(t *testing.T, scope lambdav1.Scope, role string) testIdent {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	u, err := url.Parse(scope.URI(role))
+	require.NoError(t, err)
+	usage := x509.ExtKeyUsageServerAuth
+	if role == lambdav1.RoleGuest {
+		usage = x509.ExtKeyUsageClientAuth
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: role},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{usage},
+		BasicConstraintsValid: true, URIs: []*url.URL{u},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	require.NoError(t, err)
+	kder, err := x509.MarshalECPrivateKey(key)
+	require.NoError(t, err)
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: kder})
+	c, err := tls.X509KeyPair(certPEM, keyPEM)
+	require.NoError(t, err)
+	leaf, err := x509.ParseCertificate(der)
+	require.NoError(t, err)
+	return testIdent{cert: c, leaf: leaf, certPEM: certPEM, keyPEM: keyPEM}
+}
+
+// hostConfig is the host end as the executor will run it: it pins one guest
+// certificate and checks its URI SAN with the shared pki.VerifyPinned.
+func hostConfig(host testIdent, pinGuest *x509.Certificate, scope lambdav1.Scope) *tls.Config {
+	return &tls.Config{
+		MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{host.cert},
+		ClientAuth: tls.RequireAnyClientCert, NextProtos: []string{lambdav1.ALPNv1},
+		VerifyConnection: func(cs tls.ConnectionState) error {
+			return pki.VerifyPinned(cs.PeerCertificates, pinGuest, x509.ExtKeyUsageClientAuth, scope.URI(lambdav1.RoleGuest))
+		},
+	}
+}
+
+// serve accepts one connection with cfg, writes "ok" if the handshake
+// succeeds, and returns the address and the host's handshake result.
+func serve(t *testing.T, cfg *tls.Config) (string, <-chan error) {
+	t.Helper()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	t.Cleanup(func() { ln.Close() })
+	done := make(chan error, 1)
+	go func() {
+		raw, err := ln.Accept()
+		if err != nil {
+			done <- err
+			return
+		}
+		defer raw.Close()
+		tc := tls.Server(raw, cfg)
+		_ = tc.SetDeadline(time.Now().Add(5 * time.Second))
+		err = tc.Handshake()
+		if err == nil {
+			_, err = tc.Write([]byte("ok"))
+		}
+		done <- err
+	}()
+	return ln.Addr().String(), done
+}
+
+func guestIdentity(guest, pinHost testIdent, scope lambdav1.Scope, addr string) Identity {
+	return Identity{Cert: guest.cert, HostCert: pinHost.leaf, Scope: scope, HostAddr: addr}
+}
+
+func dialAndRead(t *testing.T, id Identity) (string, error) {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	c, err := Dial(ctx, id)
+	if err != nil {
+		return "", err
+	}
+	defer c.Close()
+	_ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
+	buf := make([]byte, 2)
+	n, err := c.Read(buf)
+	return string(buf[:n]), err
+}
+
+func TestDialMutualAuthentication(t *testing.T) {
+	host, guest := newTestIdent(t, gen1, lambdav1.RoleHost), newTestIdent(t, gen1, lambdav1.RoleGuest)
+	addr, hostErr := serve(t, hostConfig(host, guest.leaf, gen1))
+
+	got, err := dialAndRead(t, guestIdentity(guest, host, gen1, addr))
+	require.NoError(t, err)
+	assert.Equal(t, "ok", got)
+	require.NoError(t, <-hostErr)
+}
+
+func TestDialRefusesHost(t *testing.T) {
+	guest := newTestIdent(t, gen1, lambdav1.RoleGuest)
+	pinned := newTestIdent(t, gen1, lambdav1.RoleHost)
+	cases := []struct {
+		name    string
+		host    testIdent
+		pin     testIdent
+		mutate  func(*tls.Config)
+		wantErr string
+	}{
+		{"unpinned host certificate", newTestIdent(t, gen1, lambdav1.RoleHost), pinned, nil, "peer certificate"},
+		{"pinned certificate of a later generation", newTestIdent(t, gen2, lambdav1.RoleHost), testIdent{}, nil, "peer identity mismatch"},
+		{"guest role presented as host", newTestIdent(t, gen1, lambdav1.RoleGuest), testIdent{}, nil, "peer certificate"},
+		{"no application protocol", pinned, pinned, func(c *tls.Config) { c.NextProtos = nil }, "application protocol"},
+		{"TLS 1.2 only", pinned, pinned, func(c *tls.Config) { c.MinVersion, c.MaxVersion = tls.VersionTLS12, tls.VersionTLS12 }, "protocol version"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			pin := tc.pin
+			if pin.leaf == nil {
+				pin = tc.host // the guest pins exactly what is presented
+			}
+			cfg := hostConfig(tc.host, guest.leaf, gen1)
+			if tc.mutate != nil {
+				tc.mutate(cfg)
+			}
+			addr, _ := serve(t, cfg)
+			_, err := dialAndRead(t, guestIdentity(guest, pin, gen1, addr))
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.wantErr)
+		})
+	}
+}
+
+// newCASignedHost returns a CA and a host leaf it signed carrying the same URI
+// SAN as a legitimate host certificate.
+func newCASignedHost(t *testing.T, scope lambdav1.Scope) (ca testIdent, leaf tls.Certificate) {
+	t.Helper()
+	caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	u, err := url.Parse(scope.URI(lambdav1.RoleHost))
+	require.NoError(t, err)
+	caTmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "ca"},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true, IsCA: true, URIs: []*url.URL{u},
+	}
+	caDER, err := x509.CreateCertificate(rand.Reader, caTmpl, caTmpl, &caKey.PublicKey, caKey)
+	require.NoError(t, err)
+	caCert, err := x509.ParseCertificate(caDER)
+	require.NoError(t, err)
+
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "host"},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		URIs: []*url.URL{u},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, &key.PublicKey, caKey)
+	require.NoError(t, err)
+	return testIdent{leaf: caCert}, tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
+}
+
+// Trusting the pin as a root is not enough: a different leaf it signed, with
+// the right URI SAN, must still be refused.
+func TestDialRefusesLeafSignedByPin(t *testing.T) {
+	guest := newTestIdent(t, gen1, lambdav1.RoleGuest)
+	ca, leaf := newCASignedHost(t, gen1)
+	for name, chain := range map[string][][]byte{
+		"leaf only":    leaf.Certificate,
+		"leaf with CA": {leaf.Certificate[0], ca.leaf.Raw},
+	} {
+		t.Run(name, func(t *testing.T) {
+			cfg := hostConfig(testIdent{}, guest.leaf, gen1)
+			cfg.Certificates = []tls.Certificate{{Certificate: chain, PrivateKey: leaf.PrivateKey}}
+			addr, _ := serve(t, cfg)
+			_, err := dialAndRead(t, guestIdentity(guest, ca, gen1, addr))
+			require.Error(t, err)
+			assert.Regexp(t, "not the pinned certificate|want exactly the pinned one", err.Error())
+		})
+	}
+}
+
+func TestHostRefusalSurfacesOnFirstRead(t *testing.T) {
+	host, guest := newTestIdent(t, gen1, lambdav1.RoleHost), newTestIdent(t, gen1, lambdav1.RoleGuest)
+	other := newTestIdent(t, gen1, lambdav1.RoleGuest)
+	addr, hostErr := serve(t, hostConfig(host, other.leaf, gen1))
+
+	_, err := dialAndRead(t, guestIdentity(guest, host, gen1, addr))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "bad certificate")
+	assert.ErrorContains(t, <-hostErr, "peer certificate")
+}
+
+func TestHostRefusesGuestOfAnotherGeneration(t *testing.T) {
+	host, guest := newTestIdent(t, gen1, lambdav1.RoleHost), newTestIdent(t, gen2, lambdav1.RoleGuest)
+	addr, hostErr := serve(t, hostConfig(host, guest.leaf, gen1))
+
+	_, err := dialAndRead(t, guestIdentity(guest, host, gen1, addr))
+	require.Error(t, err)
+	assert.ErrorIs(t, <-hostErr, pki.ErrPeerIdentity)
+}
+
+func writeEntries(t *testing.T, entries map[string][]byte) string {
+	t.Helper()
+	dir := t.TempDir()
+	for name, b := range entries {
+		require.NoError(t, os.MkdirAll(filepath.Join(dir, name), 0o700))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, name, "raw"), b, 0o600))
+	}
+	return dir
+}
+
+func validEntries(t *testing.T) map[string][]byte {
+	host, guest := newTestIdent(t, gen1, lambdav1.RoleHost), newTestIdent(t, gen1, lambdav1.RoleGuest)
+	return map[string][]byte{
+		lambdav1.EntryGuestKey: guest.keyPEM, lambdav1.EntryGuestCert: guest.certPEM, lambdav1.EntryHostCert: host.certPEM,
+		lambdav1.EntryScope: []byte(string(gen1) + "\n"), lambdav1.EntryHostAddr: []byte("10.15.8.1:18443\n"),
+	}
+}
+
+func TestLoadIdentity(t *testing.T) {
+	id, err := LoadIdentity(writeEntries(t, validEntries(t)))
+	require.NoError(t, err)
+	assert.Equal(t, gen1, id.Scope)
+	assert.Equal(t, "10.15.8.1:18443", id.HostAddr)
+	require.Len(t, id.HostCert.URIs, 1)
+	assert.Equal(t, gen1.URI(lambdav1.RoleHost), id.HostCert.URIs[0].String())
+}
+
+func TestLoadIdentityRejects(t *testing.T) {
+	other := newTestIdent(t, gen1, lambdav1.RoleGuest)
+	cases := []struct {
+		name    string
+		edit    func(map[string][]byte)
+		wantErr string
+	}{
+		{"missing entry", func(e map[string][]byte) { delete(e, lambdav1.EntryHostCert) }, "read host-cert"},
+		{"key does not match certificate", func(e map[string][]byte) { e[lambdav1.EntryGuestKey] = other.keyPEM }, "guest key pair"},
+		{"two host certificates", func(e map[string][]byte) {
+			e[lambdav1.EntryHostCert] = append(e[lambdav1.EntryHostCert], e[lambdav1.EntryHostCert]...)
+		}, "exactly one PEM"},
+		{"host certificate not PEM", func(e map[string][]byte) { e[lambdav1.EntryHostCert] = []byte("nope") }, "exactly one PEM"},
+		{"short scope", func(e map[string][]byte) { e[lambdav1.EntryScope] = []byte("ap-southeast-2/000000000001/fn-1") }, "want 5 segments"},
+		{"address without port", func(e map[string][]byte) { e[lambdav1.EntryHostAddr] = []byte("10.15.8.1") }, "host address"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			e := validEntries(t)
+			tc.edit(e)
+			_, err := LoadIdentity(writeEntries(t, e))
+			assert.ErrorContains(t, err, tc.wantErr)
+		})
+	}
+}

spinifex/agents/lambda/runtimeapi/server.go

@@ -0,0 +1,285 @@
+// Package runtimeapi serves the AWS Lambda Runtime API (version 2018-06-01)
+// inside the guest, handing one invocation at a time to the function runtime.
+package runtimeapi
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// MaxResponseBytes is the largest response payload a runtime may post.
+const MaxResponseBytes = 6291456
+
+const (
+	// ErrTypeResponseTooLarge is the function error for an oversize response.
+	ErrTypeResponseTooLarge = "Function.ResponseSizeTooLarge"
+	// ErrTypeUnhandled is the function error when the runtime names no type.
+	ErrTypeUnhandled = "Unhandled"
+
+	errTypeInvalidRequestID = "InvalidRequestID"
+	errTypeTooLarge         = "RequestEntityTooLarge"
+	errTypeInvalidRequest   = "InvalidRequest"
+
+	headerRequestID = "Lambda-Runtime-Aws-Request-Id"
+	headerDeadline  = "Lambda-Runtime-Deadline-Ms"
+	headerARN       = "Lambda-Runtime-Invoked-Function-Arn"
+	headerTraceID   = "Lambda-Runtime-Trace-Id"
+	headerErrorType = "Lambda-Runtime-Function-Error-Type"
+
+	maxInitErrorBytes = 1 << 20
+)
+
+var (
+	// ErrBusy is returned by Invoke while another invocation is in flight.
+	ErrBusy = errors.New("runtimeapi: an invocation is already in flight")
+	// ErrDeadlineExceeded is returned by Invoke when the deadline passes.
+	ErrDeadlineExceeded = errors.New("runtimeapi: invocation deadline exceeded")
+	// ErrInvalidInvocation is returned by Invoke for an invocation with no request ID.
+	ErrInvalidInvocation = errors.New("runtimeapi: invocation has no request id")
+)
+
+// Invocation is one event handed to the runtime.
+type Invocation struct {
+	RequestID   string
+	FunctionARN string
+	Deadline    time.Time
+	TraceID     string
+	Payload     []byte
+}
+
+// Result is what the runtime answered. FunctionError is empty on success and
+// otherwise the error type.
+type Result struct {
+	Payload       []byte
+	FunctionError string
+}
+
+// InitError is an initialisation failure reported by the runtime.
+type InitError struct {
+	Type string
+	Body []byte
+}
+
+type flight struct {
+	inv   Invocation
+	taken bool
+	done  chan Result
+}
+
+// Server implements the Runtime API for a single in-flight invocation.
+type Server struct {
+	mu   sync.Mutex
+	cur  *flight
+	wake chan struct{}
+
+	initOnce sync.Once
+	initDone chan struct{}
+	initErr  InitError
+}
+
+// New returns a Server with no invocation in flight.
+func New() *Server {
+	return &Server{wake: make(chan struct{}, 1), initDone: make(chan struct{})}
+}
+
+// InitFailed is closed once the runtime reports an init error.
+func (s *Server) InitFailed() <-chan struct{} { return s.initDone }
+
+// InitError returns the reported init error, if any.
+func (s *Server) InitError() (InitError, bool) {
+	select {
+	case <-s.initDone:
+		return s.initErr, true
+	default:
+		return InitError{}, false
+	}
+}
+
+// Invoke hands inv to the runtime and blocks until the runtime posts a
+// response or error, the deadline passes, or ctx ends.
+func (s *Server) Invoke(ctx context.Context, inv Invocation) (Result, error) {
+	if inv.RequestID == "" {
+		return Result{}, ErrInvalidInvocation
+	}
+	f := &flight{inv: inv, done: make(chan Result, 1)}
+	s.mu.Lock()
+	if s.cur != nil {
+		s.mu.Unlock()
+		return Result{}, ErrBusy
+	}
+	s.cur = f
+	s.mu.Unlock()
+	select {
+	case s.wake <- struct{}{}:
+	default:
+	}
+
+	timer := time.NewTimer(time.Until(inv.Deadline))
+	defer timer.Stop()
+	var cause error
+	select {
+	case res := <-f.done:
+		return res, nil
+	case <-timer.C:
+		cause = ErrDeadlineExceeded
+	case <-ctx.Done():
+		cause = ctx.Err()
+	}
+
+	s.mu.Lock()
+	if s.cur == f {
+		s.cur = nil
+		s.mu.Unlock()
+		return Result{}, cause
+	}
+	s.mu.Unlock()
+	// Completed while we were giving up; the result is already queued.
+	return <-f.done, nil
+}
+
+// Handler returns the Runtime API routes.
+func (s *Server) Handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /2018-06-01/runtime/invocation/next", s.next)
+	mux.HandleFunc("POST /2018-06-01/runtime/invocation/{id}/response", s.response)
+	mux.HandleFunc("POST /2018-06-01/runtime/invocation/{id}/error", s.invocationError)
+	mux.HandleFunc("POST /2018-06-01/runtime/init/error", s.initError)
+	return mux
+}
+
+func (s *Server) next(w http.ResponseWriter, r *http.Request) {
+	for {
+		s.mu.Lock()
+		f := s.cur
+		if f != nil && !f.taken {
+			f.taken = true
+			s.mu.Unlock()
+			h := w.Header()
+			h.Set(headerRequestID, f.inv.RequestID)
+			h.Set(headerDeadline, strconv.FormatInt(f.inv.Deadline.UnixMilli(), 10))
+			h.Set(headerARN, f.inv.FunctionARN)
+			if f.inv.TraceID != "" {
+				h.Set(headerTraceID, f.inv.TraceID)
+			}
+			_, _ = w.Write(f.inv.Payload)
+			return
+		}
+		s.mu.Unlock()
+		select {
+		case <-s.wake:
+		case <-r.Context().Done():
+			return
+		}
+	}
+}
+
+// claimable reports whether id names the delivered, uncompleted invocation.
+func (s *Server) claimable(id string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.cur != nil && s.cur.taken && s.cur.inv.RequestID == id
+}
+
+// complete finishes the invocation named id, reporting whether it was live.
+func (s *Server) complete(id string, res Result) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	f := s.cur
+	if f == nil || !f.taken || f.inv.RequestID != id {
+		return false
+	}
+	s.cur = nil
+	f.done <- res
+	return true
+}
+
+func (s *Server) response(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	if !s.claimable(id) {
+		invalidID(w, id)
+		return
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, MaxResponseBytes+1))
+	if err != nil {
+		writeError(w, http.StatusBadRequest, errTypeInvalidRequest, "read body: "+err.Error())
+		return
+	}
+	if len(body) > MaxResponseBytes {
+		msg := "Response payload size exceeded maximum allowed payload size (" + strconv.Itoa(MaxResponseBytes) + " bytes)."
+		s.complete(id, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeResponseTooLarge})
+		writeError(w, http.StatusRequestEntityTooLarge, errTypeTooLarge, msg)
+		return
+	}
+	if !s.complete(id, Result{Payload: body}) {
+		invalidID(w, id)
+		return
+	}
+	writeStatus(w)
+}
+
+func (s *Server) invocationError(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	if !s.claimable(id) {
+		invalidID(w, id)
+		return
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, MaxResponseBytes))
+	if err != nil {
+		writeError(w, http.StatusBadRequest, errTypeInvalidRequest, "read body: "+err.Error())
+		return
+	}
+	errType := r.Header.Get(headerErrorType)
+	if errType == "" {
+		errType = ErrTypeUnhandled
+	}
+	if !s.complete(id, Result{Payload: body, FunctionError: errType}) {
+		invalidID(w, id)
+		return
+	}
+	writeStatus(w)
+}
+
+func (s *Server) initError(w http.ResponseWriter, r *http.Request) {
+	body, _ := io.ReadAll(io.LimitReader(r.Body, maxInitErrorBytes))
+	s.initOnce.Do(func() {
+		s.initErr = InitError{Type: r.Header.Get(headerErrorType), Body: body}
+		close(s.initDone)
+	})
+	writeStatus(w)
+}
+
+func invalidID(w http.ResponseWriter, id string) {
+	writeError(w, http.StatusBadRequest, errTypeInvalidRequestID, "Invalid request ID: "+id)
+}
+
+func writeStatus(w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusAccepted)
+	_, _ = w.Write([]byte(`{"status":"OK"}`))
+}
+
+func writeError(w http.ResponseWriter, code int, errType, msg string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(code)
+	_, _ = w.Write(ErrorBody(msg, errType))
+}
+
+type errorPayload struct {
+	ErrorMessage string `json:"errorMessage"`
+	ErrorType    string `json:"errorType"`
+}
+
+// ErrorBody renders the Lambda error document for msg and errType.
+func ErrorBody(msg, errType string) []byte {
+	b, err := json.Marshal(errorPayload{ErrorMessage: msg, ErrorType: errType})
+	if err != nil {
+		return []byte(`{"errorType":"Runtime.Unknown"}`)
+	}
+	return b
+}

spinifex/agents/lambda/runtimeapi/server_test.go

@@ -0,0 +1,292 @@
+package runtimeapi
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const base = "/2018-06-01/runtime/"
+
+type invokeOut struct {
+	res Result
+	err error
+}
+
+func start(t *testing.T) (*Server, *httptest.Server) {
+	t.Helper()
+	s := New()
+	ts := httptest.NewServer(s.Handler())
+	t.Cleanup(ts.Close)
+	return s, ts
+}
+
+func invokeAsync(s *Server, inv Invocation) <-chan invokeOut {
+	ch := make(chan invokeOut, 1)
+	go func() {
+		res, err := s.Invoke(context.Background(), inv)
+		ch <- invokeOut{res, err}
+	}()
+	return ch
+}
+
+func post(t *testing.T, url string, body []byte, hdr map[string]string) (*http.Response, []byte) {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
+	require.NoError(t, err)
+	for k, v := range hdr {
+		req.Header.Set(k, v)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(resp.Body)
+	return resp, b
+}
+
+func get(t *testing.T, url string) (*http.Response, []byte) {
+	t.Helper()
+	resp, err := http.Get(url)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(resp.Body)
+	return resp, b
+}
+
+func inv(id string, d time.Duration) Invocation {
+	return Invocation{
+		RequestID:   id,
+		FunctionARN: "arn:aws:lambda:ap-southeast-2:000000000001:function:f",
+		Deadline:    time.Now().Add(d),
+		Payload:     []byte(`{"a":1}`),
+	}
+}
+
+func errType(t *testing.T, body []byte) string {
+	t.Helper()
+	var e map[string]string
+	require.NoError(t, json.Unmarshal(body, &e))
+	assert.NotEmpty(t, e["errorMessage"])
+	return e["errorType"]
+}
+
+func TestNextResponseRoundTrip(t *testing.T) {
+	s, ts := start(t)
+	in := inv("req-1", 2*time.Second)
+	in.TraceID = "Root=1-abc"
+	out := invokeAsync(s, in)
+
+	resp, body := get(t, ts.URL+base+"invocation/next")
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, `{"a":1}`, string(body))
+	assert.Equal(t, "req-1", resp.Header.Get("Lambda-Runtime-Aws-Request-Id"))
+	assert.Equal(t, strconv.FormatInt(in.Deadline.UnixMilli(), 10), resp.Header.Get("Lambda-Runtime-Deadline-Ms"))
+	assert.Equal(t, in.FunctionARN, resp.Header.Get("Lambda-Runtime-Invoked-Function-Arn"))
+	assert.Equal(t, "Root=1-abc", resp.Header.Get("Lambda-Runtime-Trace-Id"))
+
+	r, _ := post(t, ts.URL+base+"invocation/req-1/response", []byte(`{"sum":3}`), nil)
+	assert.Equal(t, http.StatusAccepted, r.StatusCode)
+
+	got := <-out
+	require.NoError(t, got.err)
+	assert.Equal(t, `{"sum":3}`, string(got.res.Payload))
+	assert.Empty(t, got.res.FunctionError)
+}
+
+func TestTraceHeaderOmittedWhenUnset(t *testing.T) {
+	s, ts := start(t)
+	invokeAsync(s, inv("req-1", time.Second))
+	resp, _ := get(t, ts.URL+base+"invocation/next")
+	_, present := resp.Header["Lambda-Runtime-Trace-Id"]
+	assert.False(t, present)
+}
+
+func TestErrorPostKeepsBodyAndType(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+
+	body := []byte(`{"errorMessage":"boom","errorType":"Custom"}`)
+	r, _ := post(t, ts.URL+base+"invocation/req-1/error", body, map[string]string{"Lambda-Runtime-Function-Error-Type": "Custom"})
+	assert.Equal(t, http.StatusAccepted, r.StatusCode)
+
+	got := <-out
+	require.NoError(t, got.err)
+	assert.Equal(t, "Custom", got.res.FunctionError)
+	assert.JSONEq(t, string(body), string(got.res.Payload))
+}
+
+func TestErrorPostWithoutTypeIsUnhandled(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+	post(t, ts.URL+base+"invocation/req-1/error", []byte(`{}`), nil)
+	assert.Equal(t, "Unhandled", (<-out).res.FunctionError)
+}
+
+func TestUnknownAndRepeatedIDRefused(t *testing.T) {
+	s, ts := start(t)
+
+	r, b := post(t, ts.URL+base+"invocation/nope/response", []byte("x"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+	assert.Equal(t, "InvalidRequestID", errType(t, b))
+
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+	post(t, ts.URL+base+"invocation/req-1/response", []byte("ok"), nil)
+	<-out
+
+	r, b = post(t, ts.URL+base+"invocation/req-1/response", []byte("again"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+	assert.Equal(t, "InvalidRequestID", errType(t, b))
+	r, _ = post(t, ts.URL+base+"invocation/req-1/error", []byte("{}"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+}
+
+func TestResponseNotAcceptedBeforeDelivery(t *testing.T) {
+	s, ts := start(t)
+	invokeAsync(s, inv("req-1", time.Second))
+	r, _ := post(t, ts.URL+base+"invocation/req-1/response", []byte("early"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+}
+
+func TestOversizeResponse(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 5*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+
+	r, b := post(t, ts.URL+base+"invocation/req-1/response", make([]byte, MaxResponseBytes+1), nil)
+	assert.Equal(t, http.StatusRequestEntityTooLarge, r.StatusCode)
+	assert.Equal(t, "RequestEntityTooLarge", errType(t, b))
+
+	got := <-out
+	require.NoError(t, got.err)
+	assert.Equal(t, "Function.ResponseSizeTooLarge", got.res.FunctionError)
+	assert.Equal(t, "Function.ResponseSizeTooLarge", errType(t, got.res.Payload))
+}
+
+func TestResponseAtLimitAccepted(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 5*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+	r, _ := post(t, ts.URL+base+"invocation/req-1/response", make([]byte, MaxResponseBytes), nil)
+	assert.Equal(t, http.StatusAccepted, r.StatusCode)
+	got := <-out
+	require.NoError(t, got.err)
+	assert.Len(t, got.res.Payload, MaxResponseBytes)
+}
+
+func TestDeadlineExpiryRefusesLateResponse(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 40*time.Millisecond))
+	get(t, ts.URL+base+"invocation/next")
+
+	got := <-out
+	require.ErrorIs(t, got.err, ErrDeadlineExceeded)
+
+	r, b := post(t, ts.URL+base+"invocation/req-1/response", []byte("late"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+	assert.Equal(t, "InvalidRequestID", errType(t, b))
+
+	// The slot is free again for the next invocation.
+	out = invokeAsync(s, inv("req-2", 2*time.Second))
+	resp, _ := get(t, ts.URL+base+"invocation/next")
+	assert.Equal(t, "req-2", resp.Header.Get("Lambda-Runtime-Aws-Request-Id"))
+	post(t, ts.URL+base+"invocation/req-2/response", []byte("ok"), nil)
+	require.NoError(t, (<-out).err)
+}
+
+func TestConcurrentInvokeRefused(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	require.Eventually(t, func() bool {
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		return s.cur != nil
+	}, time.Second, time.Millisecond)
+
+	_, err := s.Invoke(context.Background(), inv("req-2", time.Second))
+	require.ErrorIs(t, err, ErrBusy)
+
+	get(t, ts.URL+base+"invocation/next")
+	post(t, ts.URL+base+"invocation/req-1/response", []byte("ok"), nil)
+	require.NoError(t, (<-out).err)
+}
+
+func TestInvokeContextCancel(t *testing.T) {
+	s, _ := start(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() {
+		_, err := s.Invoke(ctx, inv("req-1", time.Minute))
+		done <- err
+	}()
+	cancel()
+	require.ErrorIs(t, <-done, context.Canceled)
+}
+
+func TestNextBlocksUntilInvoke(t *testing.T) {
+	s, ts := start(t)
+	type got struct {
+		resp *http.Response
+		body []byte
+	}
+	ch := make(chan got, 1)
+	go func() {
+		resp, err := http.Get(ts.URL + base + "invocation/next") //nolint:noctx // test client
+		if err != nil {
+			ch <- got{}
+			return
+		}
+		defer resp.Body.Close()
+		body, _ := io.ReadAll(resp.Body)
+		ch <- got{resp, body}
+	}()
+
+	select {
+	case <-ch:
+		t.Fatal("next returned before any invocation")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	g := <-ch
+	require.NotNil(t, g.resp, "next request failed")
+	assert.Equal(t, "req-1", g.resp.Header.Get("Lambda-Runtime-Aws-Request-Id"))
+	post(t, ts.URL+base+"invocation/req-1/response", []byte("ok"), nil)
+	require.NoError(t, (<-out).err)
+}
+
+func TestInitErrorRecorded(t *testing.T) {
+	s, ts := start(t)
+	_, ok := s.InitError()
+	require.False(t, ok)
+
+	r, _ := post(t, ts.URL+base+"init/error", []byte(`{"errorMessage":"bad"}`), map[string]string{"Lambda-Runtime-Function-Error-Type": "Runtime.InitError"})
+	assert.Equal(t, http.StatusAccepted, r.StatusCode)
+
+	select {
+	case <-s.InitFailed():
+	default:
+		t.Fatal("InitFailed not closed")
+	}
+	ie, ok := s.InitError()
+	require.True(t, ok)
+	assert.Equal(t, "Runtime.InitError", ie.Type)
+	assert.JSONEq(t, `{"errorMessage":"bad"}`, string(ie.Body))
+}
+
+func TestInvokeRequiresRequestID(t *testing.T) {
+	s := New()
+	_, err := s.Invoke(context.Background(), Invocation{Deadline: time.Now().Add(time.Second)})
+	require.ErrorIs(t, err, ErrInvalidInvocation)
+}

spinifex/agents/rds/agent/agent_test.go

@@ -763,10 +763,11 @@ func TestPollDBCommandsOutput_MatchesGatewayShape(t *testing.T) {
 	// either side's tags fails here.
 	payload := awsxml.QueryResponsePayload("PollDBCommands", &rdsawsapi.PollDBCommandsOutput{
 		Commands: []rdsinstance.Command{{
-			CommandID:  "cmd-1",
-			Type:       "reload-parameters",
-			Parameters: []rdsinstance.Parameter{{Name: "work_mem", Value: "8MB"}},
-			IssuedAt:   &issued,
+			CommandID:         "cmd-1",
+			Type:              "reload-parameters",
+			Parameters:        []rdsinstance.Parameter{{Name: "work_mem", Value: "8MB"}},
+			IssuedAt:          &issued,
+			RemainingBudgetMs: 29500,
 		}},
 	})
 	body, err := awsxml.Marshal(payload)
@@ -802,6 +803,9 @@ func TestPollDBCommandsOutput_MatchesGatewayShape(t *testing.T) {
 	if cmd.IssuedAt == nil || !cmd.IssuedAt.Equal(issued) {
 		t.Errorf("IssuedAt = %v, want %v", cmd.IssuedAt, issued)
 	}
+	if cmd.RemainingBudgetMs != 29500 {
+		t.Errorf("RemainingBudgetMs = %d, want 29500: a copy that loses it is granted a fresh budget", cmd.RemainingBudgetMs)
+	}
 }
 
 func readFile(t *testing.T, path string) string {

spinifex/agents/rds/agent/command.go

@@ -2,10 +2,13 @@ package agent
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
+	"slices"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	rdsinstance "github.com/mulgadc/spinifex/spinifex/domains/rds/instance"
@@ -90,35 +93,96 @@ func commandParams(cmd rdsinstance.Command) map[string]string {
 // line rate for as long as it stays broken.
 const pollErrorBackoff = 5 * time.Second
 
+// How many finished command IDs are remembered, so a copy the issuer re-published
+// while the command ran is not executed a second time.
+const completedCommandMemory = 64
+
+// How many engine commands may wait behind the one running. Past this a command
+// is refused at once rather than queued behind work its issuer will not outlive.
+const serialQueueDepth = 16
+
+// The budget for a type the control plane does not issue. Such a command is
+// answered as unsupported before it could use any of it.
+const fallbackCommandBudget = 30 * time.Second
+
+// The only command that runs beside the serial lane. It releases the backup hold
+// an engine command can be blocked on, so it must never queue behind one.
+func outOfBand(commandType string) bool {
+	return commandType == rdsinstance.CommandUnquiesce
+}
+
+// A command waiting its turn on the serial lane, with the moment its issuer
+// stops waiting for it.
+type queuedCommand struct {
+	cmd      rdsinstance.Command
+	deadline time.Time
+}
+
 // The agent's command channel: a long poll that carries back the replies for
-// whatever the previous poll delivered.
+// whatever earlier polls delivered. Engine commands run one at a time on a single
+// lane; the release of a backup hold runs beside it, so it cannot be kept out.
 type commander struct {
 	cp       controlPlane
 	id       identity
 	registry commandRegistry
 	wait     time.Duration
+	// The control plane's budget per type, capped by the remaining budget the
+	// issuer stamps on each copy and measured from receipt. IssuedAt is not
+	// compared: the guest clock is not trusted against the host's.
+	budget func(commandType string) time.Duration
+	serial chan queuedCommand
+
+	mu sync.Mutex
 	// Cleared only once a poll has carried them, so a failed poll re-delivers
 	// rather than dropping the result of work the guest actually did.
 	pending []rdsinstance.CommandReply
+	// Queued or running, so a re-published copy is not queued a second time.
+	inFlight map[string]struct{}
+	// Oldest first, bounded by completedCommandMemory.
+	completed []string
+	// Ends the poll in progress, so a reply that finished during it rides the
+	// next one rather than waiting out the window. Nil between polls.
+	interruptPoll func()
+	interrupted   bool
+	running       sync.WaitGroup
 }
 
 func newCommander(cp controlPlane, registry commandRegistry, wait time.Duration) *commander {
 	if wait <= 0 {
 		wait = defaultPollWait
 	}
-	return &commander{cp: cp, registry: registry, wait: wait}
+	return &commander{
+		cp:       cp,
+		registry: registry,
+		wait:     wait,
+		budget:   controlPlaneBudget,
+		serial:   make(chan queuedCommand, serialQueueDepth),
+		inFlight: make(map[string]struct{}),
+	}
+}
+
+func controlPlaneBudget(commandType string) time.Duration {
+	if budget, ok := rdsinstance.CommandBudget(commandType); ok {
+		return budget
+	}
+	return fallbackCommandBudget
 }
 
 func (c *commander) Run(ctx context.Context) {
+	defer c.running.Wait()
+	c.running.Go(func() { c.serveSerial(ctx) })
 	for {
 		if ctx.Err() != nil {
 			return
 		}
-		commands, err := c.cp.PollCommands(ctx, c.id, c.pending, c.wait)
+		commands, sent, err := c.poll(ctx)
 		if err != nil {
 			if ctx.Err() != nil {
 				return
 			}
+			if errors.Is(err, errReplyReady) {
+				continue
+			}
 			slog.Warn("rds-agent: command poll failed", "err", err)
 			select {
 			case <-ctx.Done():
@@ -127,14 +191,122 @@ func (c *commander) Run(ctx context.Context) {
 			}
 			continue
 		}
-		c.pending = nil
+		c.mu.Lock()
+		c.pending = c.pending[sent:]
+		c.mu.Unlock()
 
 		for _, cmd := range commands {
-			c.pending = append(c.pending, c.execute(ctx, cmd))
+			c.dispatch(ctx, cmd)
 		}
 	}
 }
 
+// Reports that a poll was ended early so a finished reply can ride the next one.
+var errReplyReady = errors.New("a command reply is ready")
+
+// One long poll carrying every reply ready when it starts. It is cut short when
+// another reply becomes ready, so a finished command is never answered a whole
+// poll window late. Returns how many pending replies the poll carried.
+func (c *commander) poll(ctx context.Context) ([]rdsinstance.Command, int, error) {
+	pollCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	c.mu.Lock()
+	replies := slices.Clone(c.pending)
+	c.interruptPoll, c.interrupted = cancel, false
+	c.mu.Unlock()
+
+	commands, err := c.cp.PollCommands(pollCtx, c.id, replies, c.wait)
+
+	c.mu.Lock()
+	interrupted := c.interrupted
+	c.interruptPoll = nil
+	c.mu.Unlock()
+	if err != nil {
+		if interrupted && ctx.Err() == nil {
+			return nil, 0, errReplyReady
+		}
+		return nil, 0, err
+	}
+	return commands, len(replies), nil
+}
+
+// Routes cmd to its lane unless it is already queued, running or answered: the
+// issuer re-publishes until it has a reply, so the same ID arrives again.
+func (c *commander) dispatch(ctx context.Context, cmd rdsinstance.Command) {
+	c.mu.Lock()
+	_, queued := c.inFlight[cmd.CommandID]
+	if queued || slices.Contains(c.completed, cmd.CommandID) {
+		c.mu.Unlock()
+		slog.Debug("rds-agent: duplicate command ignored", "commandID", cmd.CommandID, "type", cmd.Type)
+		return
+	}
+	c.inFlight[cmd.CommandID] = struct{}{}
+	c.mu.Unlock()
+
+	q := queuedCommand{cmd: cmd, deadline: time.Now().Add(c.budgetFor(cmd))}
+	if outOfBand(cmd.Type) {
+		c.running.Go(func() { c.runQueued(ctx, q) })
+		return
+	}
+	select {
+	case c.serial <- q:
+	default:
+		c.finish(cmd, failedReply(cmd, "the agent is busy; too many commands are waiting"))
+	}
+}
+
+// The type's budget, or what the issuer had left when it published this copy if
+// that is less: a copy delayed in transit must not be granted a fresh budget its
+// issuer will not wait out.
+func (c *commander) budgetFor(cmd rdsinstance.Command) time.Duration {
+	budget := c.budget(cmd.Type)
+	if cmd.RemainingBudgetMs > 0 {
+		budget = min(budget, time.Duration(cmd.RemainingBudgetMs)*time.Millisecond)
+	}
+	return budget
+}
+
+func (c *commander) serveSerial(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case q := <-c.serial:
+			c.runQueued(ctx, q)
+		}
+	}
+}
+
+// Runs a command inside what is left of its issuer's budget. One whose budget
+// ran out while it waited is answered as expired and never touches the engine,
+// since its issuer has already reported it failed.
+func (c *commander) runQueued(ctx context.Context, q queuedCommand) {
+	if !time.Now().Before(q.deadline) {
+		slog.Warn("rds-agent: command expired before it ran", "commandID", q.cmd.CommandID, "type", q.cmd.Type)
+		c.finish(q.cmd, failedReply(q.cmd, "the command expired before the agent could run it"))
+		return
+	}
+	cmdCtx, cancel := context.WithDeadline(ctx, q.deadline)
+	defer cancel()
+	c.finish(q.cmd, c.execute(cmdCtx, q.cmd))
+}
+
+func (c *commander) finish(cmd rdsinstance.Command, reply rdsinstance.CommandReply) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	delete(c.inFlight, cmd.CommandID)
+	c.completed = append(c.completed, cmd.CommandID)
+	if len(c.completed) > completedCommandMemory {
+		c.completed = c.completed[len(c.completed)-completedCommandMemory:]
+	}
+	c.pending = append(c.pending, reply)
+	if c.interruptPoll != nil {
+		c.interruptPoll()
+		c.interrupted = true
+	}
+}
+
 // Every command gets a reply: the issuer is blocked on this command ID, so a
 // silent drop costs it a full timeout.
 func (c *commander) execute(ctx context.Context, cmd rdsinstance.Command) rdsinstance.CommandReply {

spinifex/agents/rds/agent/command_test.go

@@ -3,6 +3,9 @@ package agent
 import (
 	"context"
 	"errors"
+	"fmt"
+	"slices"
+	"sync"
 	"testing"
 	"time"
 
@@ -70,9 +73,9 @@ func TestCommander_UnknownTypeRepliesFailedNotDropped(t *testing.T) {
 	}
 }
 
-// The reply for work the guest actually did rides the next poll. A poll that
+// The reply for work the guest actually did rides a later poll. A poll that
 // fails must not consume it, or the issuer never learns the outcome.
-func TestCommander_RunCarriesRepliesOnTheNextPoll(t *testing.T) {
+func TestCommander_RunCarriesRepliesOnALaterPoll(t *testing.T) {
 	cp := newFakeControlPlane()
 	cp.pollQueue = [][]rdsinstance.Command{
 		{{CommandID: "cmd-1", Type: "reload-parameters"}},
@@ -87,19 +90,15 @@ func TestCommander_RunCarriesRepliesOnTheNextPoll(t *testing.T) {
 	done := make(chan struct{})
 	go func() { c.Run(ctx); close(done) }()
 
-	waitFor(t, func() bool {
-		replies := cp.snapshotReplies()
-		return len(replies) >= 2 && len(replies[1]) == 1
-	}, "the reply to ride the second poll")
+	waitFor(t, func() bool { return len(repliesFor(cp, "cmd-1")) > 0 }, "the reply to ride a later poll")
 	cancel()
 	<-done
 
-	replies := cp.snapshotReplies()
-	if len(replies[0]) != 0 {
+	if replies := cp.snapshotReplies(); len(replies[0]) != 0 {
 		t.Errorf("first poll carried replies %+v, want none", replies[0])
 	}
-	if got := replies[1][0]; got.CommandID != "cmd-1" || got.Status != rdsinstance.CommandStatusSucceeded {
-		t.Errorf("second poll carried %+v, want the succeeded reply for cmd-1", got)
+	if got := repliesFor(cp, "cmd-1")[0]; got.Status != rdsinstance.CommandStatusSucceeded {
+		t.Errorf("the poll carried %+v, want the succeeded reply for cmd-1", got)
 	}
 }
 
@@ -141,16 +140,19 @@ func TestCommander_RetainsAndRetriesTheCompletePendingBatch(t *testing.T) {
 	done = make(chan struct{})
 	go func() { c.Run(ctx); close(done) }()
 	waitFor(t, func() bool {
-		return len(cp.snapshotReplies()) >= 3
-	}, "the retained batch to succeed and the new reply to reach the following poll")
+		return len(repliesFor(cp, "cmd-3")) > 0
+	}, "the retained batch to succeed and the new reply to reach a later poll")
 	cancel()
 	<-done
 
 	replies = cp.snapshotReplies()
 	assert.Equal(t, pending, replies[1], "the complete failed batch must be retried in order")
-	require.Len(t, replies[2], 1)
-	assert.Equal(t, "cmd-3", replies[2][0].CommandID)
-	assert.Equal(t, rdsinstance.CommandStatusSucceeded, replies[2][0].Status)
+	assert.Equal(t, rdsinstance.CommandStatusSucceeded, repliesFor(cp, "cmd-3")[0].Status)
+	for _, later := range replies[2:] {
+		for _, reply := range later {
+			assert.Equal(t, "cmd-3", reply.CommandID, "a batch a poll carried must not be sent again")
+		}
+	}
 }
 
 // A poll error must not spin the loop: without a backoff a gateway that is down
@@ -171,3 +173,340 @@ func TestCommander_BacksOffAfterAFailedPoll(t *testing.T) {
 		t.Errorf("polled %d times in 200ms against a broken channel, want 1", polls)
 	}
 }
+
+// A MariaDB statement that commits waits out a held BACKUP STAGE, and the
+// release is itself a command. Run one at a time, the release would queue behind
+// the statement it is meant to unblock until the hold's own deadline.
+func TestCommander_ABlockedCommandDoesNotKeepOutTheRelease(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollQueue = [][]rdsinstance.Command{
+		{{CommandID: "cmd-password", Type: rdsinstance.CommandSetPassword}},
+		{{CommandID: "cmd-release", Type: rdsinstance.CommandUnquiesce}},
+	}
+	released := make(chan struct{})
+	registry := commandRegistry{
+		rdsinstance.CommandSetPassword: func(ctx context.Context, _ rdsinstance.Command) (string, error) {
+			select {
+			case <-released:
+				return "", nil
+			case <-ctx.Done():
+				return "", ctx.Err()
+			}
+		},
+		rdsinstance.CommandUnquiesce: func(context.Context, rdsinstance.Command) (string, error) {
+			close(released)
+			return "", nil
+		},
+	}
+
+	c := newCommander(cp, registry, 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+
+	waitFor(t, func() bool {
+		return len(repliesFor(cp, "cmd-password")) > 0 && len(repliesFor(cp, "cmd-release")) > 0
+	}, "both the release and the command it unblocked to be answered")
+	cancel()
+	<-done
+
+	for _, id := range []string{"cmd-password", "cmd-release"} {
+		assert.Equal(t, rdsinstance.CommandStatusSucceeded, repliesFor(cp, id)[0].Status, id)
+	}
+}
+
+// The issuer re-publishes until it has a reply, so a command still running is
+// delivered again. Running the copy would rotate a password twice or open a
+// second backup alongside the first.
+func TestCommander_ARepublishedCommandRunsOnce(t *testing.T) {
+	cp := newFakeControlPlane()
+	cmd := rdsinstance.Command{CommandID: "cmd-1", Type: "reload-parameters"}
+	cp.pollQueue = [][]rdsinstance.Command{{cmd}, {cmd}}
+	var mu sync.Mutex
+	runs := 0
+	secondDelivered := make(chan struct{})
+	registry := commandRegistry{
+		"reload-parameters": func(ctx context.Context, _ rdsinstance.Command) (string, error) {
+			mu.Lock()
+			runs++
+			mu.Unlock()
+			select {
+			case <-secondDelivered:
+			case <-ctx.Done():
+			}
+			return "done", nil
+		},
+	}
+
+	c := newCommander(cp, registry, 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+
+	// The third poll means both copies have been handed to the commander.
+	waitFor(t, func() bool { return len(cp.snapshotReplies()) >= 3 }, "both copies to be delivered")
+	close(secondDelivered)
+	waitFor(t, func() bool { return len(repliesFor(cp, "cmd-1")) > 0 }, "the reply")
+	cancel()
+	<-done
+
+	mu.Lock()
+	defer mu.Unlock()
+	assert.Equal(t, 1, runs, "a re-published copy of a running command must not run again")
+}
+
+// Every reply to id carried on any poll so far.
+func repliesFor(cp *fakeControlPlane, id string) []rdsinstance.CommandReply {
+	var out []rdsinstance.CommandReply
+	for _, batch := range cp.snapshotReplies() {
+		for _, reply := range batch {
+			if reply.CommandID == id {
+				out = append(out, reply)
+			}
+		}
+	}
+	return out
+}
+
+// A copy can arrive after the handler has returned but before a poll has
+// carried its reply. The in-flight set has already let go of it then, so only
+// the completed set stops it running a second time.
+func TestCommander_ACopyArrivingBeforeTheReplyIsCarriedDoesNotRunAgain(t *testing.T) {
+	var mu sync.Mutex
+	runs := map[string]int{}
+	handler := func(_ context.Context, cmd rdsinstance.Command) (string, error) {
+		mu.Lock()
+		runs[cmd.CommandID]++
+		mu.Unlock()
+		return "", nil
+	}
+	c := newCommander(nil, commandRegistry{"reload-parameters": handler}, time.Second)
+	ctx := t.Context()
+	go c.serveSerial(ctx)
+
+	first := rdsinstance.Command{CommandID: "cmd-1", Type: "reload-parameters"}
+	c.dispatch(ctx, first)
+	waitFor(t, func() bool { return len(c.pendingReplies()) == 1 }, "the first run's reply to be pending")
+
+	c.dispatch(ctx, first)
+	// The lane is FIFO, so once this one has answered a queued copy would have run.
+	c.dispatch(ctx, rdsinstance.Command{CommandID: "cmd-2", Type: "reload-parameters"})
+	waitFor(t, func() bool {
+		return slices.ContainsFunc(c.pendingReplies(), func(r rdsinstance.CommandReply) bool { return r.CommandID == "cmd-2" })
+	}, "the following command's reply")
+
+	mu.Lock()
+	defer mu.Unlock()
+	assert.Equal(t, 1, runs["cmd-1"], "a copy arriving before the reply was carried must not run")
+	assert.Len(t, c.pendingReplies(), 2, "the copy must not be answered a second time either")
+}
+
+// A quiesce that waited out its issuer's budget behind another command would
+// put the engine in backup mode for a snapshot nobody is taking.
+func TestCommander_AStaleQuiesceNeverTakesAHold(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollQueue = [][]rdsinstance.Command{
+		{{CommandID: "cmd-apply", Type: rdsinstance.CommandApplyParams}},
+		{{CommandID: "cmd-quiesce", Type: rdsinstance.CommandQuiesce, Parameters: []rdsinstance.Parameter{
+			{Name: rdsinstance.CommandParamQuiesceLabel, Value: "orders-db-automated"},
+			{Name: rdsinstance.CommandParamQuiesceDeadlineSeconds, Value: "360"},
+		}}},
+	}
+	engine := &fakeEngine{}
+	registry := newCommandRegistry(engine, &fakeStorage{})
+	applying, release := make(chan struct{}), make(chan struct{})
+	registry[rdsinstance.CommandApplyParams] = func(ctx context.Context, _ rdsinstance.Command) (string, error) {
+		close(applying)
+		<-release
+		return "", nil
+	}
+	c := newCommander(cp, registry, 10*time.Millisecond)
+	c.budget = func(commandType string) time.Duration {
+		if commandType == rdsinstance.CommandQuiesce {
+			return 50 * time.Millisecond
+		}
+		return controlPlaneBudget(commandType)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+
+	<-applying
+	waitFor(t, func() bool { return len(cp.snapshotReplies()) >= 3 }, "the quiesce to be delivered behind the apply")
+	time.Sleep(100 * time.Millisecond)
+	close(release)
+	waitFor(t, func() bool { return len(repliesFor(cp, "cmd-quiesce")) > 0 }, "the quiesce to be answered")
+	cancel()
+	<-done
+
+	reply := repliesFor(cp, "cmd-quiesce")[0]
+	assert.Equal(t, rdsinstance.CommandStatusFailed, reply.Status)
+	assert.Contains(t, reply.Message, "expired")
+	assert.Empty(t, engine.label, "an expired quiesce must never reach the engine")
+}
+
+// Engine commands still run one at a time: a parameter apply rewrites the
+// option file a graceful stop would otherwise race.
+func TestCommander_AParameterApplyNeverRunsAlongsideAnEngineStop(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollQueue = [][]rdsinstance.Command{
+		{{CommandID: "cmd-apply", Type: rdsinstance.CommandApplyParams}},
+		{{CommandID: "cmd-stop", Type: rdsinstance.CommandStopEngine}},
+	}
+	var mu sync.Mutex
+	active, most := 0, 0
+	release := make(chan struct{})
+	// Each holds the lane until released, so two running at once are both seen
+	// whichever is scheduled first.
+	engineWork := func(context.Context, rdsinstance.Command) (string, error) {
+		mu.Lock()
+		active++
+		most = max(most, active)
+		mu.Unlock()
+		<-release
+		mu.Lock()
+		active--
+		mu.Unlock()
+		return "", nil
+	}
+	registry := commandRegistry{
+		rdsinstance.CommandApplyParams: engineWork,
+		rdsinstance.CommandStopEngine:  engineWork,
+	}
+	c := newCommander(cp, registry, 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+
+	waitFor(t, func() bool { return len(cp.snapshotReplies()) >= 3 }, "the stop to be delivered behind the apply")
+	time.Sleep(50 * time.Millisecond)
+	close(release)
+	waitFor(t, func() bool { return len(repliesFor(cp, "cmd-stop")) > 0 }, "the stop to be answered")
+	cancel()
+	<-done
+
+	mu.Lock()
+	defer mu.Unlock()
+	assert.Equal(t, 1, most, "the engine stop ran while the parameter apply was still running")
+}
+
+// The agent bounds each command by its issuer's budget, so a type the control
+// plane has no budget for would run on a guess.
+func TestCommandRegistry_EveryTypeHasAControlPlaneBudget(t *testing.T) {
+	for commandType := range newCommandRegistry(&fakeEngine{}, &fakeStorage{}) {
+		if _, ok := rdsinstance.CommandBudget(commandType); !ok {
+			t.Errorf("%s has no control-plane budget", commandType)
+		}
+	}
+}
+
+func (c *commander) pendingReplies() []rdsinstance.CommandReply {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return slices.Clone(c.pending)
+}
+
+// The first copy can reach the agent late, after a gateway or bus outage, with
+// its issuer about to give up. A quiesce granted a fresh budget then would take
+// a hold nobody is left to release.
+func TestCommander_AQuiesceStampedNearlyOutOfBudgetNeverTakesAHold(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollQueue = [][]rdsinstance.Command{
+		{{CommandID: "cmd-apply", Type: rdsinstance.CommandApplyParams}},
+		{{CommandID: "cmd-quiesce", Type: rdsinstance.CommandQuiesce, RemainingBudgetMs: 50, Parameters: []rdsinstance.Parameter{
+			{Name: rdsinstance.CommandParamQuiesceLabel, Value: "orders-db-automated"},
+			{Name: rdsinstance.CommandParamQuiesceDeadlineSeconds, Value: "360"},
+		}}},
+	}
+	engine := &fakeEngine{}
+	registry := newCommandRegistry(engine, &fakeStorage{})
+	applying, release := make(chan struct{}), make(chan struct{})
+	registry[rdsinstance.CommandApplyParams] = func(context.Context, rdsinstance.Command) (string, error) {
+		close(applying)
+		<-release
+		return "", nil
+	}
+	c := newCommander(cp, registry, 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+
+	<-applying
+	waitFor(t, func() bool { return len(cp.snapshotReplies()) >= 3 }, "the quiesce to be delivered behind the apply")
+	// The quiesce was received before the third poll began, so its stamped budget
+	// has run out 50ms after this point.
+	queuedAt := time.Now()
+	waitFor(t, func() bool { return time.Since(queuedAt) > 50*time.Millisecond }, "the stamped budget to pass")
+	close(release)
+	waitFor(t, func() bool { return len(repliesFor(cp, "cmd-quiesce")) > 0 }, "the quiesce to be answered")
+	cancel()
+	<-done
+
+	reply := repliesFor(cp, "cmd-quiesce")[0]
+	assert.Equal(t, rdsinstance.CommandStatusFailed, reply.Status)
+	assert.Contains(t, reply.Message, "expired")
+	assert.Empty(t, engine.label, "a quiesce whose issuer had all but given up must never reach the engine")
+}
+
+// A full lane refuses at once rather than queueing work its issuer will not
+// outlive, and the refusal is the command's answer: a re-published copy must
+// not slip in once the lane has room.
+func TestCommander_AFullLaneRefusesAtOnceAndNeverRunsTheCommand(t *testing.T) {
+	var mu sync.Mutex
+	runs := map[string]int{}
+	running, release := make(chan struct{}, 1), make(chan struct{})
+	handler := func(_ context.Context, cmd rdsinstance.Command) (string, error) {
+		mu.Lock()
+		runs[cmd.CommandID]++
+		mu.Unlock()
+		if cmd.CommandID == "cmd-blocker" {
+			running <- struct{}{}
+			<-release
+		}
+		return "", nil
+	}
+	c := newCommander(nil, commandRegistry{"reload-parameters": handler}, time.Second)
+	ctx := t.Context()
+	go c.serveSerial(ctx)
+
+	c.dispatch(ctx, rdsinstance.Command{CommandID: "cmd-blocker", Type: "reload-parameters"})
+	<-running
+	for i := range serialQueueDepth {
+		c.dispatch(ctx, rdsinstance.Command{CommandID: fmt.Sprintf("cmd-queued-%d", i), Type: "reload-parameters"})
+	}
+
+	overflow := rdsinstance.Command{CommandID: "cmd-overflow", Type: "reload-parameters"}
+	c.dispatch(ctx, overflow)
+	refused := slices.IndexFunc(c.pendingReplies(), func(r rdsinstance.CommandReply) bool { return r.CommandID == overflow.CommandID })
+	require.GreaterOrEqual(t, refused, 0, "a full lane must answer at once, before anything ahead of it finishes")
+	assert.Equal(t, rdsinstance.CommandStatusFailed, c.pendingReplies()[refused].Status)
+	assert.Contains(t, c.pendingReplies()[refused].Message, "busy")
+
+	close(release)
+	last := fmt.Sprintf("cmd-queued-%d", serialQueueDepth-1)
+	waitFor(t, func() bool {
+		return slices.ContainsFunc(c.pendingReplies(), func(r rdsinstance.CommandReply) bool { return r.CommandID == last })
+	}, "the queue to drain")
+	c.dispatch(ctx, overflow)
+	// FIFO again: once this answers, a copy let onto the lane would have run.
+	c.dispatch(ctx, rdsinstance.Command{CommandID: "cmd-after", Type: "reload-parameters"})
+	waitFor(t, func() bool {
+		return slices.ContainsFunc(c.pendingReplies(), func(r rdsinstance.CommandReply) bool { return r.CommandID == "cmd-after" })
+	}, "the command after the re-published copy")
+
+	mu.Lock()
+	defer mu.Unlock()
+	assert.Zero(t, runs[overflow.CommandID], "a refused command must never run")
+	answers := 0
+	for _, r := range c.pendingReplies() {
+		if r.CommandID == overflow.CommandID {
+			answers++
+		}
+	}
+	assert.Equal(t, 1, answers, "a re-published copy of a refused command must be ignored, not answered again")
+}

spinifex/agents/rds/agent/engine_mariadb.go

@@ -94,6 +94,8 @@ type mariadbEngine struct {
 	socket    string
 	rcService string
 	service   string
+	// How long a password rotation waits for a backup hold to end.
+	holdWait time.Duration
 }
 
 var _ engine = (*mariadbEngine)(nil)
@@ -151,6 +153,7 @@ func newMariaDBEngine(cfg Config, rules controlPlaneRules, run commandRunner, st
 		socket:    mariadbSocketPath(cfg),
 		rcService: cfg.RCService,
 		service:   cfg.EngineService,
+		holdWait:  mariadbPasswordHoldWait,
 		parameterManager: parameterManager{
 			probe: probe,
 			params: parameterStore{
@@ -380,6 +383,11 @@ SET SESSION slow_query_log = 0;
 SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';
 `
 
+// ALTER USER can take effect in memory and then wait at its commit for a held
+// BACKUP STAGE to end. Waiting for the agent's own hold before the statement, for
+// less than the control plane's budget, keeps that hold from half-applying it.
+const mariadbPasswordHoldWait = 20 * time.Second
+
 // Rotates the master's password. The statement is built here rather than by the
 // client, which has neither psql's identifier interpolation nor its parameter
 // quoting, and it rides stdin so it is never visible in the process table.
@@ -395,16 +403,34 @@ func (e *mariadbEngine) SetPassword(ctx context.Context, username, password stri
 		return fmt.Errorf("refusing to set the password of role %q: %w", username, err)
 	}
 
+	// Only the agent's own backup hold is waited out before the statement. A lock
+	// someone else holds, such as a customer FTWRL, is met inside it and ended by
+	// lock_wait_timeout, by which point the password may already have changed.
 	escaped := sqlLiteral(password)
-	sql := fmt.Sprintf("%sALTER USER '%s'@'%s' IDENTIFIED BY '%s';\n",
-		mariadbSessionGuard, username, mariadbMasterHost, escaped)
-	if _, err := e.clientRun(ctx, sql); err != nil {
-		// The client echoes the failing statement, which carries the escaped form
-		// of the password rather than the raw one — so redacting only the raw one
-		// would leak exactly the passwords that needed escaping.
-		return fmt.Errorf("apply the master password: %s", redact(redact(err.Error(), password), escaped))
+	return e.outsideHold(ctx, e.holdWait, func() error {
+		// Computed after the hold wait, so the statement gets only what is left.
+		sql := fmt.Sprintf("%sSET SESSION lock_wait_timeout = %d;\nALTER USER '%s'@'%s' IDENTIFIED BY '%s';\n",
+			mariadbSessionGuard, statementLockWait(ctx), username, mariadbMasterHost, escaped)
+		if _, err := e.clientRun(ctx, sql); err != nil {
+			// The client echoes the failing statement, which carries the escaped form
+			// of the password rather than the raw one — so redacting only the raw one
+			// would leak exactly the passwords that needed escaping.
+			return fmt.Errorf("apply the master password: %s", redact(redact(err.Error(), password), escaped))
+		}
+		return nil
+	})
+}
+
+// The server-side lock wait for a statement, in whole seconds: what remains of
+// ctx less a margin for the reply, so the server gives up its wait before the
+// client is killed. A ctx with no deadline gets the full password budget.
+func statementLockWait(ctx context.Context) int {
+	const replyMargin = 2 * time.Second
+	remaining := mariadbPasswordHoldWait
+	if deadline, ok := ctx.Deadline(); ok {
+		remaining = time.Until(deadline) - replyMargin
 	}
-	return nil
+	return max(1, int(remaining.Seconds()))
 }
 
 // Installs the resolved set and applies the half a running server can adopt.
@@ -714,7 +740,9 @@ func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Dur
 
 	// Held across the whole start, so a second quiesce waits and then finds the
 	// first one's hold rather than opening a concurrent backup alongside it.
-	e.mu.Lock()
+	if err := e.lockWithin(ctx); err != nil {
+		return err
+	}
 	defer e.mu.Unlock()
 	if e.held != nil {
 		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)

spinifex/agents/rds/agent/engine_mariadb_test.go

@@ -1151,3 +1151,123 @@ func TestMariaDBEngine_EveryResolvedDynamicValueRenders(t *testing.T) {
 		}
 	}
 }
+
+// ALTER USER under a held BACKUP STAGE swaps the password in memory and then
+// waits at its commit, so a rotation issued mid-backup must not reach the
+// engine until the hold is released.
+func TestMariaDBEngine_SetPasswordWaitsForTheBackupHoldToEnd(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+	engine.startSess = func(context.Context, command) (engineSession, error) { return &fakeSession{}, nil }
+	if err := engine.Quiesce(context.Background(), "orders-db-automated", time.Minute); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+
+	rotated := make(chan error, 1)
+	go func() { rotated <- engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw") }()
+	select {
+	case err := <-rotated:
+		t.Fatalf("SetPassword returned %v while the engine was held for a backup", err)
+	case <-time.After(3 * holdPollInterval):
+	}
+
+	if err := engine.Unquiesce(context.Background()); err != nil {
+		t.Fatalf("Unquiesce: %v", err)
+	}
+	select {
+	case err := <-rotated:
+		if err != nil {
+			t.Fatalf("SetPassword after the release: %v", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("SetPassword did not run once the hold was released")
+	}
+	if len(runner.calls) != 1 || !strings.Contains(runner.calls[0].Stdin, "ALTER USER") {
+		t.Errorf("ran %d commands, want the one rotation after the release", len(runner.calls))
+	}
+}
+
+// A hold that outlasts the wait fails the rotation before anything is sent, so
+// the control plane's error is the truth rather than half of it.
+func TestMariaDBEngine_SetPasswordFailsUntouchedWhenTheHoldOutlastsItsWait(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+	engine.holdWait = 3 * holdPollInterval
+	engine.startSess = func(context.Context, command) (engineSession, error) { return &fakeSession{}, nil }
+	if err := engine.Quiesce(context.Background(), "orders-db-automated", time.Minute); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err == nil {
+		t.Fatal("SetPassword succeeded while the engine stayed held for a backup")
+	}
+	if len(runner.calls) != 0 {
+		t.Errorf("ran %d commands against a held engine, want none", len(runner.calls))
+	}
+}
+
+// The wait has to end before the control plane gives up on the command, or the
+// rotation can still land after the caller was told it failed.
+func TestMariaDBEngine_PasswordHoldWaitFitsInsideTheControlPlaneTimeout(t *testing.T) {
+	budget, ok := rdsinstance.CommandBudget(rdsinstance.CommandSetPassword)
+	if !ok {
+		t.Fatal("the control plane has no budget for set-password")
+	}
+	if mariadbPasswordHoldWait >= budget-5*time.Second {
+		t.Errorf("a %s hold wait leaves under 5s of the control plane's %s set-password budget for the statement", mariadbPasswordHoldWait, budget)
+	}
+}
+
+// The statement is bounded by what is left of the command, so the server gives
+// up a lock wait before the client is killed under it.
+func TestMariaDBEngine_SetPasswordBoundsTheStatementByTheRemainingBudget(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	if err := engine.SetPassword(ctx, "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	sql := runner.calls[0].Stdin
+	at := strings.Index(sql, "SET SESSION lock_wait_timeout = ")
+	if at < 0 || at > strings.Index(sql, "ALTER USER") {
+		t.Fatalf("SQL %q does not bound the statement's lock wait before it runs", sql)
+	}
+	var seconds int
+	if _, err := fmt.Sscanf(sql[at:], "SET SESSION lock_wait_timeout = %d;", &seconds); err != nil {
+		t.Fatalf("parse the lock wait: %v", err)
+	}
+	if seconds < 1 || seconds > 8 {
+		t.Errorf("lock wait = %ds with 10s of budget left, want it inside the budget less the reply margin", seconds)
+	}
+}
+
+// A quiesce that cannot get the hold bookkeeping must give up with its command,
+// not wait on the mutex after its issuer has moved on.
+func TestMariaDBEngine_QuiesceGivesUpWaitingForTheBookkeepingWithItsContext(t *testing.T) {
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	started := 0
+	engine.startSess = func(context.Context, command) (engineSession, error) {
+		started++
+		return &fakeSession{}, nil
+	}
+	engine.mu.Lock()
+	defer engine.mu.Unlock()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 3*holdPollInterval)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- engine.Quiesce(ctx, "orders-db-automated", time.Minute) }()
+	select {
+	case err := <-done:
+		if err == nil {
+			t.Fatal("Quiesce succeeded without the bookkeeping lock")
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Quiesce waited on the bookkeeping past its context")
+	}
+	if started != 0 {
+		t.Errorf("opened %d backup sessions, want none", started)
+	}
+}

spinifex/agents/rds/agent/engine_postgres.go

@@ -516,7 +516,9 @@ func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Du
 
 	// Held across the whole start, so a second quiesce waits and then finds the
 	// first one's hold rather than opening a concurrent backup alongside it.
-	e.mu.Lock()
+	if err := e.lockWithin(ctx); err != nil {
+		return err
+	}
 	defer e.mu.Unlock()
 	if e.held != nil {
 		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)

spinifex/agents/rds/agent/quiesce.go

@@ -114,6 +114,48 @@ func (q *quiesceState) releaseHold(ctx context.Context, releaseSQL string) error
 	return nil
 }
 
+// Takes mu, giving up when ctx ends. A quiesce that cannot get the bookkeeping
+// inside its budget must fail rather than start a backup its issuer abandoned.
+func (q *quiesceState) lockWithin(ctx context.Context) error {
+	for !q.mu.TryLock() {
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("wait for the backup bookkeeping: %w", ctx.Err())
+		case <-time.After(holdPollInterval):
+		}
+	}
+	return nil
+}
+
+// How often a statement waiting for a backup hold to end looks again.
+const holdPollInterval = 100 * time.Millisecond
+
+// Runs fn while no backup hold is on the engine, and keeps a new one from
+// starting until it returns. A hold that outlasts wait fails the call before fn
+// has touched the engine.
+func (q *quiesceState) outsideHold(ctx context.Context, wait time.Duration, fn func() error) error {
+	deadline := time.NewTimer(wait)
+	defer deadline.Stop()
+	for {
+		// Not Lock: a quiesce holds mu while its stages wait on their own locks,
+		// and this wait has to stay bounded by ctx and wait.
+		if q.mu.TryLock() {
+			if q.held == nil {
+				defer q.mu.Unlock()
+				return fn()
+			}
+			q.mu.Unlock()
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-deadline.C:
+			return fmt.Errorf("the engine is still held for a backup after %s", wait)
+		case <-time.After(holdPollInterval):
+		}
+	}
+}
+
 // The deadline. Ending the session is enough — both engines release the hold
 // with it — so this never has to reach the engine itself.
 func (q *quiesceState) expire(label string) {

spinifex/bootstrap/config/config.go

@@ -5,6 +5,7 @@ package config
 import (
 	"fmt"
 	"log/slog"
+	"maps"
 	"net/netip"
 	"os"
 	"path/filepath"
@@ -355,6 +356,11 @@ type EBSConfig struct {
 	CacheSizeMB *int `json:"CacheSizeMB" mapstructure:"cache_size_mb"`
 }
 
+// SupportedEBSProvider reports whether name is a provider a node may run.
+func SupportedEBSProvider(name string) bool {
+	return name == EBSProviderViperblockd || name == EBSProviderQEMUNBD
+}
+
 // ResolvedProvider normalizes an empty Provider to EBSProviderViperblockd,
 // the default provider.
 func (c EBSConfig) ResolvedProvider() string {
@@ -801,6 +807,27 @@ func validateVultrPool(cc *ClusterConfig, p ExternalPool) error {
 	return nil
 }
 
+// validateSingleEBSProvider refuses nodes whose explicit [nodes.<name>.ebs]
+// providers differ. A peer stanza without one is unknown, not the default: the
+// generated config omits it, and the cluster's KV lock is the authoritative check.
+func validateSingleEBSProvider(cc *ClusterConfig) error {
+	var first, want string
+	for _, name := range slices.Sorted(maps.Keys(cc.Nodes)) {
+		if !viper.IsSet("nodes." + name + ".ebs.provider") {
+			continue
+		}
+		got := cc.Nodes[name].EBS.ResolvedProvider()
+		if first == "" {
+			first, want = name, got
+			continue
+		}
+		if got != want {
+			return fmt.Errorf("config: [nodes.%s.ebs] provider resolves to %q but [nodes.%s.ebs] resolves to %q; every node in a cluster must use the same EBS provider", first, want, name, got)
+		}
+	}
+	return nil
+}
+
 func validateClusterConfig(cc *ClusterConfig) error {
 	if viper.IsSet("network.external_dhcp") {
 		return fmt.Errorf("config: [network] external_dhcp is no longer supported; remove the key (static WAN-pool allocation only)")
@@ -809,9 +836,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		if viper.IsSet("nodes." + nodeName + ".vpcd.dhcp_bind_bridge") {
 			return fmt.Errorf("config: [nodes.%s.vpcd] dhcp_bind_bridge is no longer supported; remove the key (vpcd no longer runs a DHCP client)", nodeName)
 		}
-		switch nodeCfg.EBS.Provider {
-		case "", EBSProviderViperblockd, EBSProviderQEMUNBD:
-		case EBSProviderEmbedded:
+		switch {
+		case nodeCfg.EBS.Provider == "", SupportedEBSProvider(nodeCfg.EBS.Provider):
+		case nodeCfg.EBS.Provider == EBSProviderEmbedded:
 			// Refuse rather than silently upgrade: the embedded engine is gone,
 			// and starting anyway would migrate this node's volumes to
 			// ebsmetadata one-way without the operator ever asking for it.
@@ -829,6 +856,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 			return fmt.Errorf("config: [nodes.%s.ebs] cache_size_mb=%d must not be negative; use 0 to disable the cache, or remove the key for the default of %d", nodeName, *c, DefaultCacheSizeMB)
 		}
 	}
+	if err := validateSingleEBSProvider(cc); err != nil {
+		return err
+	}
 
 	if err := validateNorthstarCluster(cc); err != nil {
 		return err

spinifex/bootstrap/config/config_test.go

@@ -1108,6 +1108,45 @@ provider = "bogus"
 	assert.Contains(t, err.Error(), `provider="bogus"`)
 }
 
+func TestLoadConfig_EBSProvider_SingleProviderPerCluster(t *testing.T) {
+	stanza := func(node, provider string) string {
+		return fmt.Sprintf("[nodes.%s.ebs]\nprovider = %q\n", node, provider)
+	}
+	cases := []struct {
+		name    string
+		ebs     []string
+		wantErr string
+	}{
+		{"all unset", []string{"", "", ""}, ""},
+		{"explicit viperblockd beside unset peers", []string{"", stanza("n2", EBSProviderViperblockd), ""}, ""},
+		{"explicit qemunbd beside unset peers", []string{stanza("n1", EBSProviderQEMUNBD), "", ""}, ""},
+		{"all explicit qemunbd", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderQEMUNBD), stanza("n3", EBSProviderQEMUNBD)}, ""},
+		{"two explicit providers differ", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderViperblockd), ""},
+			`[nodes.n1.ebs] provider resolves to "qemunbd" but [nodes.n2.ebs] resolves to "viperblockd"`},
+		{"only the third node differs", []string{stanza("n1", EBSProviderViperblockd), stanza("n2", EBSProviderViperblockd), stanza("n3", EBSProviderQEMUNBD)},
+			`[nodes.n1.ebs] provider resolves to "viperblockd" but [nodes.n3.ebs] resolves to "qemunbd"`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			body := "node = \"n1\"\n"
+			for i, ebs := range tc.ebs {
+				body += fmt.Sprintf("\n[nodes.n%d]\nregion = \"us-east-1\"\n%s", i+1, ebs)
+			}
+			resetViper(t)
+			path := filepath.Join(t.TempDir(), "spinifex.toml")
+			require.NoError(t, os.WriteFile(path, []byte(body), 0600))
+
+			_, err := LoadConfig(path)
+			if tc.wantErr == "" {
+				require.NoError(t, err)
+				return
+			}
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.wantErr)
+		})
+	}
+}
+
 func TestEBSConfig_ResolvedTunablesDefault(t *testing.T) {
 	var c EBSConfig
 	assert.Equal(t, DefaultNBDKitThreads, c.ResolvedThreads())

spinifex/domains/ec2/accountcleanup/cleanup.go

@@ -69,6 +69,28 @@ var (
 	_ TagIndexOwner       = (*ec2tags.TagsServiceImpl)(nil)
 )
 
+// kindPrefixes maps each cleanable kind to its resource-ID prefix in the tag index.
+var kindPrefixes = map[string]string{
+	ec2v1.PlatformKindKeyPair:        keyPairIDPrefix,
+	ec2v1.PlatformKindPlacementGroup: placementGroupIDPrefix,
+	ec2v1.PlatformKindLaunchTemplate: launchTemplateIDPrefix,
+}
+
+// selectKinds returns the kinds a request cleans, in report order. No kinds
+// means all three; a kind that is not cleanable, such as the tag index, is refused.
+func selectKinds(requested []string) ([]string, error) {
+	all := []string{ec2v1.PlatformKindKeyPair, ec2v1.PlatformKindPlacementGroup, ec2v1.PlatformKindLaunchTemplate}
+	if len(requested) == 0 {
+		return all, nil
+	}
+	for _, kind := range requested {
+		if !slices.Contains(all, kind) {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The kind is not one of key-pair, placement-group or launch-template.")
+		}
+	}
+	return slices.DeleteFunc(all, func(kind string) bool { return !slices.Contains(requested, kind) }), nil
+}
+
 // DefaultSettleWindow is how old headerless residue must be before it is
 // removed; anything younger may be a create still in flight.
 const DefaultSettleWindow = 10 * time.Minute
@@ -102,6 +124,7 @@ const (
 // CleanupPlatform deletes the account's key pairs, empty placement groups and
 // launch templates, removes settled residue and orphaned tag-index entries,
 // then re-scans every owner store to build the result. It is safe to re-run.
+// Request.Kinds limits the run to those kinds and their tag-index entries.
 func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, error) {
 	if req == nil || req.AccountID == "" {
 		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "The request must specify an account ID.")
@@ -118,27 +141,54 @@ func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, e
 	acct := req.AccountID
 	res := &Result{AccountID: acct, CorrelationRef: req.CorrelationRef}
 
-	refs := c.readReferences(ctx, acct)
-	keyRun := c.cleanKeyPairs(ctx, acct, settledBefore, refs)
-	groupRun := c.cleanPlacementGroups(ctx, acct, refs)
-	templateRun := c.cleanLaunchTemplates(ctx, acct, settledBefore)
+	kinds, err := selectKinds(req.Kinds)
+	if err != nil {
+		return nil, err
+	}
+	var prefixes []string
+	for _, kind := range kinds {
+		prefixes = append(prefixes, kindPrefixes[kind])
+	}
+
+	var keyRun *keyPairRun
+	var groupRun *placementGroupRun
+	var templateRun *launchTemplateRun
+	if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) || slices.Contains(kinds, ec2v1.PlatformKindPlacementGroup) {
+		refs := c.readReferences(ctx, acct)
+		if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) {
+			keyRun = c.cleanKeyPairs(ctx, acct, settledBefore, refs)
+		}
+		if slices.Contains(kinds, ec2v1.PlatformKindPlacementGroup) {
+			groupRun = c.cleanPlacementGroups(ctx, acct, refs)
+		}
+	}
+	if slices.Contains(kinds, ec2v1.PlatformKindLaunchTemplate) {
+		templateRun = c.cleanLaunchTemplates(ctx, acct, settledBefore)
+	}
 
 	// Tags are listed before the owner re-scans, so a resource created in
 	// between is seen as owned and keeps its tags.
 	tagKind := c.startKind(ec2v1.PlatformKindTagIndex)
-	tagged, tagErr := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	tagged, tagErr := c.tags.ScanAccountResourceIDs(ctx, acct, prefixes...)
 	if tagErr != nil {
 		slog.WarnContext(ctx, "Platform cleanup: tag index listing failed", "err", tagErr)
 		tagKind.incomplete(ec2v1.PlatformReasonStoreListFailed)
 	}
 
-	owners := map[string]ownedIDs{
-		keyPairIDPrefix:        c.finishKeyPairs(ctx, acct, keyRun, res),
-		placementGroupIDPrefix: c.finishPlacementGroups(ctx, acct, groupRun, res),
-		launchTemplateIDPrefix: c.finishLaunchTemplates(ctx, acct, templateRun, res),
+	owners := map[string]ownedIDs{}
+	if keyRun != nil {
+		owners[keyPairIDPrefix] = c.finishKeyPairs(ctx, acct, keyRun, res)
+		res.Kinds = append(res.Kinds, keyRun.kind.report)
 	}
-	res.Kinds = []KindReport{keyRun.kind.report, groupRun.kind.report, templateRun.kind.report}
-	res.Kinds = append(res.Kinds, c.cleanTagIndex(ctx, acct, tagKind, tagged, tagErr == nil, owners, res))
+	if groupRun != nil {
+		owners[placementGroupIDPrefix] = c.finishPlacementGroups(ctx, acct, groupRun, res)
+		res.Kinds = append(res.Kinds, groupRun.kind.report)
+	}
+	if templateRun != nil {
+		owners[launchTemplateIDPrefix] = c.finishLaunchTemplates(ctx, acct, templateRun, res)
+		res.Kinds = append(res.Kinds, templateRun.kind.report)
+	}
+	res.Kinds = append(res.Kinds, c.cleanTagIndex(ctx, acct, tagKind, tagged, prefixes, tagErr == nil, owners, res))
 
 	slog.InfoContext(ctx, "Platform cleanup completed", "accountID", acct, "correlationRef", req.CorrelationRef,
 		"items", len(res.Items), "duration_ms", otelsetup.Millis(time.Since(start)))

spinifex/domains/ec2/accountcleanup/cleanup_test.go

@@ -775,3 +775,38 @@ func (f failingResidue) CleanupAccountResidue(ctx context.Context, acct string,
 	}
 	return &ec2launchtemplate.AccountCleanup{KeptTemplateIDs: scan.LiveTemplateIDs, OrphanBodiesFailed: scan.OrphanVersionBodies}, nil
 }
+
+// A run limited to one kind cleans that kind and its tag-index entries only:
+// the other kinds keep their resources and their tags, and are not reported.
+func TestCleanupPlatformLimitedToKindsLeavesTheRestAlone(t *testing.T) {
+	o := newOwners(t)
+	keyID := o.importKey(t, testAccount, "k1")
+	groupID := o.createGroup(t, testAccount, "g1")
+	o.tag(t, testAccount, keyID)
+
+	c := o.newCleanup()
+	c.now = func() time.Time { return time.Now().Add(time.Hour) }
+	res, err := c.CleanupPlatform(t.Context(), &Request{AccountID: testAccount, Kinds: []string{ec2v1.PlatformKindPlacementGroup}})
+	require.NoError(t, err)
+
+	assert.Equal(t, []Item{item(groupID, ec2v1.PlatformKindPlacementGroup, DispositionDeleted, "")}, res.Items)
+	assert.Equal(t, map[string]string{
+		ec2v1.PlatformKindPlacementGroup: ec2v1.PlatformStatusObservedClean,
+		ec2v1.PlatformKindTagIndex:       ec2v1.PlatformStatusObservedClean,
+	}, statuses(res))
+
+	keys, err := o.keys.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, keys.LiveKeyPairIDs, "an unselected kind is not touched")
+	tagged, err := o.tags.ScanAccountResourceIDs(t.Context(), testAccount, "key-")
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, tagged, "an unselected kind keeps its tags")
+}
+
+func TestCleanupPlatformRefusesAKindItCannotClean(t *testing.T) {
+	o := newOwners(t)
+	for _, kinds := range [][]string{{ec2v1.PlatformKindTagIndex}, {"volume"}} {
+		_, err := o.newCleanup().CleanupPlatform(t.Context(), &Request{AccountID: testAccount, Kinds: kinds})
+		require.Error(t, err, "%v", kinds)
+	}
+}

spinifex/domains/ec2/accountcleanup/kinds.go

@@ -282,15 +282,15 @@ func (c *Cleanup) finishLaunchTemplates(ctx context.Context, acct string, run *l
 // cleanTagIndex clears entries whose owner re-scan resolved and holds no
 // record for the ID, then re-lists the index to prove each clear. Entries of
 // resources that still exist are theirs, not residue, and are left alone.
-func (c *Cleanup) cleanTagIndex(ctx context.Context, acct string, kind *kindRun, tagged []string, listed bool, owners map[string]ownedIDs, res *Result) KindReport {
+func (c *Cleanup) cleanTagIndex(ctx context.Context, acct string, kind *kindRun, tagged, prefixes []string, listed bool, owners map[string]ownedIDs, res *Result) KindReport {
 	if listed {
-		c.clearOrphanTags(ctx, acct, kind, tagged, owners, res)
+		c.clearOrphanTags(ctx, acct, kind, tagged, prefixes, owners, res)
 	}
 	c.finishKind(kind)
 	return kind.report
 }
 
-func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRun, tagged []string, owners map[string]ownedIDs, res *Result) {
+func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRun, tagged, prefixes []string, owners map[string]ownedIDs, res *Result) {
 	// A resource this run reports as not deleted owns its tags even when its
 	// kind's re-scan failed.
 	reported := map[string]bool{}
@@ -317,7 +317,7 @@ func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRu
 			cleared = append(cleared, id)
 		}
 	}
-	after, err := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	after, err := c.tags.ScanAccountResourceIDs(ctx, acct, prefixes...)
 	if err != nil {
 		slog.WarnContext(ctx, "Platform cleanup: tag index re-scan failed", "err", err)
 		kind.incomplete(ec2v1.PlatformReasonStoreListFailed)

spinifex/domains/ec2/accountcleanup/types.go

@@ -7,6 +7,9 @@ import "time"
 type Request struct {
 	AccountID      string `json:"account_id"`
 	CorrelationRef string `json:"correlation_ref,omitempty"`
+
+	// Kinds limits the run to these platform kinds. Empty means all three.
+	Kinds []string `json:"kinds,omitempty"`
 }
 
 // Result is what the owner stores hold after one run. Kinds reuse the ec2.v1

spinifex/domains/ec2/instance/service_nats.go

@@ -1,80 +0,0 @@
-package instance
-
-import (
-	"context"
-	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/nats-io/nats.go"
-)
-
-// NATSInstanceService handles instance operations via NATS. Fan-out methods
-// (DescribeInstances, DescribeInstanceTypes) intentionally error — use the
-// gateway's scatter-gather helper for cluster-wide results.
-type NATSInstanceService struct {
-	natsConn *nats.Conn
-}
-
-var _ InstanceService = (*NATSInstanceService)(nil)
-
-// NewNATSInstanceService creates a new NATS-based instance service.
-func NewNATSInstanceService(conn *nats.Conn) InstanceService {
-	return &NATSInstanceService{natsConn: conn}
-}
-
-func (s *NATSInstanceService) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
-	if input == nil || input.InstanceType == nil {
-		return nil, fmt.Errorf("instance type is required")
-	}
-	topic := fmt.Sprintf("ec2.RunInstances.%s", aws.StringValue(input.InstanceType))
-	return natsmsg.NATSRequest[ec2.Reservation](ctx, s.natsConn, topic, input, 5*time.Minute, accountID)
-}
-
-// DescribeInstances is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstances(_ context.Context, _ *ec2.DescribeInstancesInput, _ string) (*ec2.DescribeInstancesOutput, error) {
-	return nil, fmt.Errorf("DescribeInstances is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-// DescribeInstanceTypes is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstanceTypes(_ context.Context, _ *ec2.DescribeInstanceTypesInput, _ string) (*DescribeInstanceTypesNodeOutput, error) {
-	return nil, fmt.Errorf("DescribeInstanceTypes is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-// DescribeInstanceStatus is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstanceStatus(_ context.Context, _ *ec2.DescribeInstanceStatusInput, _ string) (*ec2.DescribeInstanceStatusOutput, error) {
-	return nil, fmt.Errorf("DescribeInstanceStatus is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-func (s *NATSInstanceService) DescribeInstanceAttribute(ctx context.Context, input *ec2.DescribeInstanceAttributeInput, accountID string) (*ec2.DescribeInstanceAttributeOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstanceAttributeOutput](ctx, s.natsConn, "ec2.DescribeInstanceAttribute", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) DescribeStoppedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstancesOutput](ctx, s.natsConn, "ec2.DescribeStoppedInstances", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) DescribeTerminatedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstancesOutput](ctx, s.natsConn, "ec2.DescribeTerminatedInstances", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) ModifyInstanceAttribute(ctx context.Context, input *ec2.ModifyInstanceAttributeInput, accountID string) (*ec2.ModifyInstanceAttributeOutput, error) {
-	return natsmsg.NATSRequest[ec2.ModifyInstanceAttributeOutput](ctx, s.natsConn, "ec2.ModifyInstanceAttribute", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) ModifyInstanceMetadataOptions(ctx context.Context, input *ec2.ModifyInstanceMetadataOptionsInput, accountID string) (*ec2.ModifyInstanceMetadataOptionsOutput, error) {
-	return natsmsg.NATSRequest[ec2.ModifyInstanceMetadataOptionsOutput](ctx, s.natsConn, "ec2.ModifyInstanceMetadataOptions", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) StartStoppedInstance(ctx context.Context, input *StartStoppedInstanceInput, accountID string) (*StartStoppedInstanceOutput, error) {
-	return natsmsg.NATSRequest[StartStoppedInstanceOutput](ctx, s.natsConn, "ec2.StartStoppedInstance", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) TerminateStoppedInstance(ctx context.Context, input *TerminateStoppedInstanceInput, accountID string) (*TerminateStoppedInstanceOutput, error) {
-	return natsmsg.NATSRequest[TerminateStoppedInstanceOutput](ctx, s.natsConn, "ec2.TerminateStoppedInstance", input, 30*time.Second, accountID)
-}

spinifex/domains/ec2/instance/service_nats_test.go

@@ -1,364 +0,0 @@
-package instance
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/nats-io/nats-server/v2/server"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// startTestNATSServer starts an embedded NATS server for testing.
-func startTestNATSServer(t *testing.T) (*server.Server, string) {
-	t.Helper()
-	ns, nc := testutil.StartTestNATS(t)
-	_ = nc // connection managed by testutil cleanup
-	return ns, ns.ClientURL()
-}
-
-// createValidRunInstancesInput creates a valid RunInstancesInput for testing.
-func createValidRunInstancesInput() *ec2.RunInstancesInput {
-	return &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String("t3.micro"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-		KeyName:      aws.String("test-key-pair"),
-		SecurityGroupIds: []*string{
-			aws.String("sg-0123456789abcdef0"),
-		},
-		SubnetId: aws.String("subnet-6e7f829e"),
-		UserData: aws.String("#!/bin/bash\necho 'test'"),
-	}
-}
-
-// createValidReservation creates a valid ec2.Reservation response for testing.
-func createValidReservation() *ec2.Reservation {
-	return &ec2.Reservation{
-		ReservationId: aws.String("r-0123456789abcdef0"),
-		OwnerId:       aws.String("123456789012"),
-		Instances: []*ec2.Instance{
-			{
-				InstanceId:   aws.String("i-0123456789abcdef0"),
-				InstanceType: aws.String("t3.micro"),
-				ImageId:      aws.String("ami-0abcdef1234567890"),
-				State: &ec2.InstanceState{
-					Code: aws.Int64(0),
-					Name: aws.String("pending"),
-				},
-				PrivateIpAddress: aws.String("10.0.1.100"),
-				SubnetId:         aws.String("subnet-6e7f829e"),
-			},
-		},
-	}
-}
-
-// TestNATSInstanceService_RunInstances_Success tests successful RunInstances operation.
-func TestNATSInstanceService_RunInstances_Success(t *testing.T) {
-	// Skip if LOG_IGNORE is set
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	// Start test NATS server
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	// Create NATS connections
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err, "Failed to connect to NATS")
-	defer nc.Close()
-
-	// Create mock daemon subscriber that responds with valid reservation
-	// Subscribe to the per-instance-type topic (t3.micro from createValidRunInstancesInput)
-	mockReservation := createValidReservation()
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		// Validate that request is properly formatted
-		var input ec2.RunInstancesInput
-		err := json.Unmarshal(msg.Data, &input)
-		if err != nil {
-			t.Errorf("Mock daemon received invalid JSON: %v", err)
-			return
-		}
-
-		// Send back valid reservation
-		responseData, _ := json.Marshal(mockReservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err, "Failed to subscribe mock daemon")
-
-	// Create NATSInstanceService
-	service := NewNATSInstanceService(nc)
-	require.NotNil(t, service)
-
-	// Create valid input
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify success
-	require.NoError(t, err, "RunInstances should succeed")
-	require.NotNil(t, reservation, "Reservation should not be nil")
-
-	// Verify reservation contents
-	assert.Equal(t, *mockReservation.ReservationId, *reservation.ReservationId)
-	assert.Equal(t, *mockReservation.OwnerId, *reservation.OwnerId)
-	require.Len(t, reservation.Instances, 1)
-
-	instance := reservation.Instances[0]
-	assert.Equal(t, "i-0123456789abcdef0", *instance.InstanceId)
-	assert.Equal(t, "t3.micro", *instance.InstanceType)
-	assert.Equal(t, int64(0), *instance.State.Code)
-	assert.Equal(t, "pending", *instance.State.Name)
-}
-
-// TestNATSInstanceService_RunInstances_DaemonError tests error handling when daemon returns error.
-func TestNATSInstanceService_RunInstances_DaemonError(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that responds with error
-	// Subscribe to the per-instance-type topic for the requested type
-	_, err = nc.QueueSubscribe("ec2.RunInstances.invalid.type", "spinifex-workers", func(msg *nats.Msg) {
-		// Send back error response
-		errorResponse := awserrors.GenerateErrorPayload("InvalidInstanceType")
-		msg.Respond(errorResponse)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-
-	input := &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String("invalid.type"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-	}
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify error handling - error should be just the AWS error code for gateway lookup
-	require.Error(t, err, "Should return error when daemon returns error")
-	assert.Nil(t, reservation, "Reservation should be nil on error")
-	assert.Equal(t, "InvalidInstanceType", err.Error(), "Error should be the AWS error code")
-}
-
-// TestNATSInstanceService_RunInstances_NoSubscriber tests behavior when no daemon is subscribed.
-func TestNATSInstanceService_RunInstances_NoSubscriber(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// No subscriber - no daemon running
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	start := time.Now()
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-	duration := time.Since(start)
-
-	// Verify error behavior
-	// When no subscribers are available, NATS returns ErrNoResponders immediately
-	require.Error(t, err, "Should fail when no subscriber")
-	assert.Nil(t, reservation, "Reservation should be nil")
-	assert.ErrorIs(t, err, nats.ErrNoResponders, "Error should be ErrNoResponders")
-
-	// Should fail quickly (within 1 second), not wait for 30 second timeout
-	assert.Less(t, duration.Seconds(), 1.0, "Should fail immediately when no responders")
-}
-
-// TestNATSInstanceService_RunInstances_InvalidResponse tests handling of malformed responses.
-func TestNATSInstanceService_RunInstances_InvalidResponse(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that responds with invalid JSON
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		// Send back malformed JSON
-		msg.Respond([]byte(`{"invalid": json response`))
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify error handling
-	require.Error(t, err, "Should return error for invalid JSON")
-	assert.Nil(t, reservation, "Reservation should be nil")
-	assert.Contains(t, err.Error(), "failed to unmarshal", "Error should indicate unmarshal failure")
-}
-
-// TestNATSInstanceService_RunInstances_MarshalError tests handling when input marshaling fails.
-func TestNATSInstanceService_RunInstances_MarshalError(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	service := NewNATSInstanceService(nc)
-
-	// Nil input should be rejected before hitting NATS
-	reservation, err := service.RunInstances(context.Background(), nil, "123456789012")
-	require.Error(t, err, "Should handle nil input")
-	assert.Nil(t, reservation)
-	assert.Contains(t, err.Error(), "instance type is required")
-}
-
-// TestNATSInstanceService_RunInstances_MultipleInstances tests launching multiple instances.
-func TestNATSInstanceService_RunInstances_MultipleInstances(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that returns multiple instances
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		var input ec2.RunInstancesInput
-		json.Unmarshal(msg.Data, &input)
-
-		// Create reservation with count matching MinCount
-		count := int(*input.MinCount)
-		instances := make([]*ec2.Instance, count)
-		for i := range count {
-			instanceId := aws.String(aws.StringValue(aws.String(fmt.Sprintf("i-%d", i))))
-			instances[i] = &ec2.Instance{
-				InstanceId:   instanceId,
-				InstanceType: input.InstanceType,
-				ImageId:      input.ImageId,
-				State: &ec2.InstanceState{
-					Code: aws.Int64(0),
-					Name: aws.String("pending"),
-				},
-			}
-		}
-
-		reservation := &ec2.Reservation{
-			ReservationId: aws.String("r-0123456789abcdef0"),
-			Instances:     instances,
-		}
-
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-
-	// Request 3 instances
-	input := createValidRunInstancesInput()
-	input.MinCount = aws.Int64(3)
-	input.MaxCount = aws.Int64(3)
-
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify multiple instances returned
-	require.NoError(t, err)
-	require.NotNil(t, reservation)
-	assert.Len(t, reservation.Instances, 3, "Should have 3 instances")
-}
-
-// TestNATSInstanceService_RunInstances_QueueGroup tests load balancing with queue groups.
-func TestNATSInstanceService_RunInstances_QueueGroup(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create multiple mock daemon workers in same queue group
-	worker1Count := 0
-	worker2Count := 0
-
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		worker1Count++
-		reservation := createValidReservation()
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		worker2Count++
-		reservation := createValidReservation()
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Send multiple requests
-	requestCount := 10
-	for range requestCount {
-		reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-		require.NoError(t, err)
-		require.NotNil(t, reservation)
-	}
-
-	// Verify load was distributed (both workers should have handled some requests)
-	// With queue groups, NATS distributes messages across subscribers
-	t.Logf("Worker 1 handled: %d requests", worker1Count)
-	t.Logf("Worker 2 handled: %d requests", worker2Count)
-
-	assert.Positive(t, worker1Count, "Worker 1 should handle some requests")
-	assert.Positive(t, worker2Count, "Worker 2 should handle some requests")
-	assert.Equal(t, requestCount, worker1Count+worker2Count, "All requests should be handled")
-}

spinifex/domains/ecr/awsapi/gateway_image_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -25,7 +26,8 @@ import (
 // object store + memory meta) and its composed ECR JSON image-action service.
 func newImageGateway(t *testing.T) *ecrGW {
 	t.Helper()
-	meta := ecrdomain.NewMemoryMetaStore()
+	_, _, js := testutil.StartTestJetStream(t)
+	meta := ecrdomain.NewKVMetaStore(js)
 	reg := ecrregistry.NewRegistry(objectstore.NewMemoryObjectStore(), meta, ecrTestAccount)
 	gw := newECRGW(nil, awsapi.Deps{Registry: awsapi.NewRegistryActionService(reg, reg, reg, reg), LifecyclePreview: awsapi.NewLifecyclePreviewActionService(meta, reg)})
 	imageRegistries.Store(gw, reg)

spinifex/domains/ecr/meta.go

@@ -5,8 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"maps"
-	"slices"
 	"sort"
 	"strings"
 	"sync"
@@ -593,295 +591,6 @@ func (s *KVMetaStore) DeleteUpload(ctx context.Context, accountID, uploadID stri
 	return nil
 }
 
-// MemoryMetaStore is an in-memory MetaStore used by tests and single-process
-// dev runs. It is safe for concurrent use.
-type MemoryMetaStore struct {
-	mu        sync.Mutex
-	repos     map[string]map[string]RepoMeta     // account -> repo -> meta
-	policies  map[string]map[string][]byte       // account -> repo -> policyText
-	lifecycle map[string]map[string][]byte       // account -> repo -> lifecyclePolicyText
-	tags      map[string]map[string]string       // account -> repo|tag -> digest
-	manifests map[string]map[string]ManifestMeta // account -> repo|digest -> meta
-	uploads   map[string]map[string]uploadRev    // account -> id -> state+rev
-}
-
-type uploadRev struct {
-	state UploadState
-	rev   uint64
-}
-
-var _ MetaStore = (*MemoryMetaStore)(nil)
-
-// NewMemoryMetaStore constructs an empty in-memory MetaStore.
-func NewMemoryMetaStore() *MemoryMetaStore {
-	return &MemoryMetaStore{
-		repos:     make(map[string]map[string]RepoMeta),
-		policies:  make(map[string]map[string][]byte),
-		lifecycle: make(map[string]map[string][]byte),
-		tags:      make(map[string]map[string]string),
-		manifests: make(map[string]map[string]ManifestMeta),
-		uploads:   make(map[string]map[string]uploadRev),
-	}
-}
-
-func (m *MemoryMetaStore) PutRepo(_ context.Context, accountID string, meta RepoMeta) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.repos[accountID] == nil {
-		m.repos[accountID] = make(map[string]RepoMeta)
-	}
-	m.repos[accountID][meta.Name] = meta
-	return nil
-}
-
-func (m *MemoryMetaStore) GetRepo(_ context.Context, accountID, repo string) (RepoMeta, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	r, ok := m.repos[accountID][repo]
-	if !ok {
-		return RepoMeta{}, ErrNotFound
-	}
-	return r, nil
-}
-
-func (m *MemoryMetaStore) ListRepos(_ context.Context, accountID string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return slices.Sorted(maps.Keys(m.repos[accountID])), nil
-}
-
-func (m *MemoryMetaStore) DeleteRepo(_ context.Context, accountID, repo string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if _, ok := m.repos[accountID][repo]; !ok {
-		return ErrNotFound
-	}
-	delete(m.repos[accountID], repo)
-	delete(m.policies[accountID], repo)
-	delete(m.lifecycle[accountID], repo)
-	for k := range m.tags[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo {
-			delete(m.tags[accountID], k)
-		}
-	}
-	for k := range m.manifests[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo {
-			delete(m.manifests[accountID], k)
-		}
-	}
-	return nil
-}
-
-func (m *MemoryMetaStore) ListManifests(_ context.Context, accountID, repo string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	var out []string
-	for k := range m.manifests[accountID] {
-		if r, d, ok := strings.Cut(k, "|"); ok && r == repo {
-			out = append(out, d)
-		}
-	}
-	sort.Strings(out)
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutRepoPolicy(_ context.Context, accountID, repo string, policyText []byte) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.policies[accountID] == nil {
-		m.policies[accountID] = make(map[string][]byte)
-	}
-	m.policies[accountID][repo] = policyText
-	return nil
-}
-
-func (m *MemoryMetaStore) GetRepoPolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.policies[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	return p, nil
-}
-
-func (m *MemoryMetaStore) DeleteRepoPolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.policies[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	delete(m.policies[accountID], repo)
-	return p, nil
-}
-
-func (m *MemoryMetaStore) PutLifecyclePolicy(_ context.Context, accountID, repo string, policyText []byte) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.lifecycle[accountID] == nil {
-		m.lifecycle[accountID] = make(map[string][]byte)
-	}
-	m.lifecycle[accountID][repo] = policyText
-	return nil
-}
-
-func (m *MemoryMetaStore) GetLifecyclePolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.lifecycle[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	return p, nil
-}
-
-func (m *MemoryMetaStore) DeleteLifecyclePolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.lifecycle[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	delete(m.lifecycle[accountID], repo)
-	return p, nil
-}
-
-func (m *MemoryMetaStore) PutTag(_ context.Context, accountID, repo, tag, digest string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.tags[accountID] == nil {
-		m.tags[accountID] = make(map[string]string)
-	}
-	m.tags[accountID][repo+"|"+tag] = digest
-	return nil
-}
-
-func (m *MemoryMetaStore) GetTag(_ context.Context, accountID, repo, tag string) (string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	d, ok := m.tags[accountID][repo+"|"+tag]
-	if !ok {
-		return "", ErrNotFound
-	}
-	return d, nil
-}
-
-func (m *MemoryMetaStore) DeleteTag(_ context.Context, accountID, repo, tag string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	key := repo + "|" + tag
-	if _, ok := m.tags[accountID][key]; !ok {
-		return ErrNotFound
-	}
-	delete(m.tags[accountID], key)
-	return nil
-}
-
-func (m *MemoryMetaStore) ListTags(_ context.Context, accountID, repo string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	var out []string
-	for k := range m.tags[accountID] {
-		if r, t, ok := strings.Cut(k, "|"); ok && r == repo {
-			out = append(out, t)
-		}
-	}
-	sort.Strings(out)
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutManifestMeta(_ context.Context, accountID, repo string, meta ManifestMeta) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.manifests[accountID] == nil {
-		m.manifests[accountID] = make(map[string]ManifestMeta)
-	}
-	m.manifests[accountID][repo+"|"+meta.Digest] = meta
-	return nil
-}
-
-func (m *MemoryMetaStore) GetManifestMeta(_ context.Context, accountID, repo, digest string) (ManifestMeta, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	meta, ok := m.manifests[accountID][repo+"|"+digest]
-	if !ok {
-		return ManifestMeta{}, ErrNotFound
-	}
-	return meta, nil
-}
-
-func (m *MemoryMetaStore) DeleteManifestMeta(_ context.Context, accountID, repo, digest string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	key := repo + "|" + digest
-	if _, ok := m.manifests[accountID][key]; !ok {
-		return ErrNotFound
-	}
-	delete(m.manifests[accountID], key)
-	return nil
-}
-
-func (m *MemoryMetaStore) ListReferrers(_ context.Context, accountID, repo, subjectDigest string) ([]ReferrerDescriptor, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	out := []ReferrerDescriptor{}
-	for k, meta := range m.manifests[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo && meta.SubjectDigest == subjectDigest {
-			out = append(out, meta.referrerDescriptor())
-		}
-	}
-	sort.Slice(out, func(i, j int) bool { return out[i].Digest < out[j].Digest })
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutUpload(_ context.Context, accountID, uploadID string, state UploadState) (uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.uploads[accountID] == nil {
-		m.uploads[accountID] = make(map[string]uploadRev)
-	}
-	rev := m.uploads[accountID][uploadID].rev + 1
-	m.uploads[accountID][uploadID] = uploadRev{state: state, rev: rev}
-	return rev, nil
-}
-
-func (m *MemoryMetaStore) GetUpload(_ context.Context, accountID, uploadID string) (UploadState, uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	u, ok := m.uploads[accountID][uploadID]
-	if !ok {
-		return UploadState{}, 0, ErrNotFound
-	}
-	return u.state, u.rev, nil
-}
-
-func (m *MemoryMetaStore) UpdateUpload(_ context.Context, accountID, uploadID string, state UploadState, rev uint64) (uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	u, ok := m.uploads[accountID][uploadID]
-	if !ok {
-		return 0, ErrNotFound
-	}
-	if u.rev != rev {
-		return 0, ErrConflict
-	}
-	u.state = state
-	u.rev = rev + 1
-	m.uploads[accountID][uploadID] = u
-	return u.rev, nil
-}
-
-func (m *MemoryMetaStore) DeleteUpload(_ context.Context, accountID, uploadID string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if _, ok := m.uploads[accountID][uploadID]; !ok {
-		return ErrNotFound
-	}
-	delete(m.uploads[accountID], uploadID)
-	return nil
-}
-
 // mapKVErr normalizes JetStream KV errors to the MetaStore error vocabulary.
 func mapKVErr(err error) error {
 	if errors.Is(err, jetstream.ErrKeyNotFound) {

spinifex/domains/ecr/meta_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -61,8 +62,8 @@ func TestKVKeyHelpers(t *testing.T) {
 	assert.Equal(t, "sha256-abc", DigestToken("sha256:abc"))
 }
 
-func TestMemoryMetaStore_RepoRoundtrip(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_RepoRoundtrip(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	_, err := m.GetRepo(context.Background(), acct, "team/app")
@@ -80,8 +81,8 @@ func TestMemoryMetaStore_RepoRoundtrip(t *testing.T) {
 	assert.Equal(t, []string{"team/app", "team/web"}, repos)
 }
 
-func TestMemoryMetaStore_TagsAndManifests(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_TagsAndManifests(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	require.NoError(t, m.PutTag(context.Background(), acct, "r", "v1", "sha256:aaa"))
@@ -112,8 +113,8 @@ func TestMemoryMetaStore_TagsAndManifests(t *testing.T) {
 	assert.ErrorIs(t, m.DeleteManifestMeta(context.Background(), acct, "r", "sha256:ccc"), ErrNotFound)
 }
 
-func TestMemoryMetaStore_UploadCAS(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_UploadCAS(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	rev, err := m.PutUpload(context.Background(), acct, "u1", UploadState{RepoName: "r"})
@@ -143,8 +144,8 @@ func TestTrimSuffixMeta(t *testing.T) {
 	assert.Equal(t, "team/app", trimSuffixMeta("repos/team/app/meta"))
 }
 
-func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_DeleteRepoAndListManifests(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	require.NoError(t, m.PutRepo(context.Background(), acct, RepoMeta{Name: "team/app", CreatedAt: time.Now()}))
@@ -156,7 +157,7 @@ func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
 
 	digs, err := m.ListManifests(context.Background(), acct, "team/app")
 	require.NoError(t, err)
-	assert.Equal(t, []string{"sha256:ccc"}, digs)
+	assert.Equal(t, []string{DigestToken("sha256:ccc")}, digs)
 
 	require.NoError(t, m.DeleteRepo(context.Background(), acct, "team/app"))
 	_, err = m.GetRepo(context.Background(), acct, "team/app")
@@ -180,50 +181,6 @@ func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
 	assert.ErrorIs(t, m.DeleteRepo(context.Background(), acct, "team/ghost"), ErrNotFound)
 }
 
-func TestMemoryMetaStore_RepoPolicy(t *testing.T) {
-	m := NewMemoryMetaStore()
-	const acct = "000000000000"
-	const policy = `{"Version":"2012-10-17"}`
-
-	_, err := m.GetRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-	_, err = m.DeleteRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-
-	require.NoError(t, m.PutRepoPolicy(context.Background(), acct, "team/app", []byte(policy)))
-	got, err := m.GetRepoPolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(got))
-
-	deleted, err := m.DeleteRepoPolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(deleted))
-	_, err = m.GetRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-}
-
-func TestMemoryMetaStore_LifecyclePolicy(t *testing.T) {
-	m := NewMemoryMetaStore()
-	const acct = "000000000000"
-	const policy = `{"rules":[]}`
-
-	_, err := m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-	_, err = m.DeleteLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-
-	require.NoError(t, m.PutLifecyclePolicy(context.Background(), acct, "team/app", []byte(policy)))
-	got, err := m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(got))
-
-	deleted, err := m.DeleteLifecyclePolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(deleted))
-	_, err = m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-}
-
 func TestManifestMeta_DecodesRecordWithoutReferrerFields(t *testing.T) {
 	var m ManifestMeta
 	require.NoError(t, json.Unmarshal([]byte(`{"digest":"sha256:ccc","mediaType":"application/vnd.oci.image.manifest.v1+json","size":10,"pushedAt":"2026-01-01T00:00:00Z","childDigests":["sha256:ddd"]}`), &m))
@@ -232,3 +189,9 @@ func TestManifestMeta_DecodesRecordWithoutReferrerFields(t *testing.T) {
 	assert.Empty(t, m.SubjectDigest)
 	assert.Empty(t, m.ArtifactType)
 }
+
+func newTestKVMetaStore(t *testing.T) *KVMetaStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	return NewKVMetaStore(js)
+}

spinifex/domains/ecr/referrers_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -30,11 +29,7 @@ func referrerMeta(digest, subject string) ManifestMeta {
 // its own referrers state.
 func referrerStores(t *testing.T) map[string]MetaStore {
 	t.Helper()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	return map[string]MetaStore{
-		"memory": NewMemoryMetaStore(),
-		"kv":     NewKVMetaStore(testutil.NewJetStream(t, nc)),
-	}
+	return map[string]MetaStore{"kv": newTestKVMetaStore(t)}
 }
 
 func TestListReferrers(t *testing.T) {

spinifex/domains/ecr/registry/lifecycle_sweep_test.go

@@ -50,7 +50,7 @@ func putPolicy(t *testing.T, reg *Registry, account, repo, policy string) {
 const untaggedExpirePolicy = `{"rules":[{"rulePriority":1,"selection":{"tagStatus":"untagged","countType":"sinceImagePushed","countUnit":"days","countNumber":7},"action":{"type":"expire"}}]}`
 
 func TestSweepRepo_SinceImagePushed(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -65,7 +65,7 @@ func TestSweepRepo_SinceImagePushed(t *testing.T) {
 }
 
 func TestSweepRepo_ImageCountMoreThan(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:a", now.AddDate(0, 0, -1), "a")
@@ -81,7 +81,7 @@ func TestSweepRepo_ImageCountMoreThan(t *testing.T) {
 }
 
 func TestSweepRepo_NoPolicy(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:x", now.AddDate(0, 0, -100))
@@ -92,7 +92,7 @@ func TestSweepRepo_NoPolicy(t *testing.T) {
 }
 
 func TestSweepRepo_Idempotent(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -105,7 +105,7 @@ func TestSweepRepo_Idempotent(t *testing.T) {
 }
 
 func TestSweepOnce_MultiAccount(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	const acct2 = "000000000001"
 	policy := `{"rules":[{"rulePriority":1,"selection":{"tagStatus":"any","countType":"imageCountMoreThan","countNumber":1},"action":{"type":"expire"}}]}`
@@ -125,7 +125,7 @@ func TestSweepOnce_MultiAccount(t *testing.T) {
 }
 
 func TestSweepOnce_AccountsError(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	s := NewLifecycleSweeper(reg, func() ([]string, error) {
 		return nil, assert.AnError
 	}, time.Hour)
@@ -133,7 +133,7 @@ func TestSweepOnce_AccountsError(t *testing.T) {
 }
 
 func TestNewLifecycleSweeper_DefaultInterval(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	s := NewLifecycleSweeper(reg, func() ([]string, error) { return nil, nil }, 0)
 	assert.Equal(t, DefaultLifecycleSweepInterval, s.interval)
 
@@ -147,7 +147,7 @@ func TestNewLifecycleSweeper_DefaultInterval(t *testing.T) {
 // TestSweeperRun drives the ticker loop: with a tiny interval and an expiring
 // image, the first tick must delete it; cancelling the context stops Run.
 func TestSweeperRun(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -184,7 +184,7 @@ func TestSweeperRun(t *testing.T) {
 // TestSweepRepo_InvalidStoredPolicy exercises the evaluate-error branch: a
 // malformed stored policy is logged and skipped, deleting nothing.
 func TestSweepRepo_InvalidStoredPolicy(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:x", now.AddDate(0, 0, -100))
@@ -198,7 +198,7 @@ func TestSweepRepo_InvalidStoredPolicy(t *testing.T) {
 // TestSweepRepo_PolicyWithoutRepo exercises the ListImages-not-found branch: a
 // policy stored for a repo with no manifests yields nothing to expire.
 func TestSweepRepo_PolicyWithoutRepo(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	putPolicy(t, reg, testAccount, "ghost", untaggedExpirePolicy)
 

spinifex/domains/ecr/registry/registry_ensurebucket_test.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"testing"
 
-	"github.com/mulgadc/spinifex/spinifex/domains/ecr"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -32,7 +31,7 @@ func (c *countingStore) EnsureBucket(ctx context.Context, bucket string) error {
 
 func TestRegistry_EnsureBucketFailureReturns500(t *testing.T) {
 	store := &countingStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), ensureErr: errors.New("predastore down")}
-	reg := NewRegistry(store, ecr.NewMemoryMetaStore(), testAccount)
+	reg := NewRegistry(store, newTestMetaStore(t), testAccount)
 
 	w := do(reg, http.MethodGet, "/v2/myrepo/tags/list", nil, nil)
 	assert.Equal(t, http.StatusInternalServerError, w.Code)
@@ -40,7 +39,7 @@ func TestRegistry_EnsureBucketFailureReturns500(t *testing.T) {
 
 func TestRegistry_EnsureBucketCachedAfterSuccess(t *testing.T) {
 	store := &countingStore{MemoryObjectStore: objectstore.NewMemoryObjectStore()}
-	reg := NewRegistry(store, ecr.NewMemoryMetaStore(), testAccount)
+	reg := NewRegistry(store, newTestMetaStore(t), testAccount)
 
 	do(reg, http.MethodGet, "/v2/myrepo/tags/list", nil, nil)
 	do(reg, http.MethodGet, "/v2/other/tags/list", nil, nil)

spinifex/domains/ecr/registry/registry_image_test.go

@@ -44,7 +44,7 @@ func storeLayersManifest(t *testing.T, reg *Registry, repo, tag string, layers .
 }
 
 func TestDeleteImage_ReclaimsBlobsKeepsShared(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	shared := pushBlob(t, reg, repo, []byte("shared-layer-bytes"))
 	exclusive := pushBlob(t, reg, repo, []byte("exclusive-layer-bytes"))
@@ -85,7 +85,7 @@ func seedImage(t *testing.T, reg *Registry, repo, tag string) ([]byte, string) {
 }
 
 func TestStoreAndGetManifest_RoundTrip(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	manifest, digest := seedImage(t, reg, "team/app", "v1")
 
 	// Fetch by tag.
@@ -114,7 +114,7 @@ func TestStoreAndGetManifest_RoundTrip(t *testing.T) {
 }
 
 func TestListImages_TaggedAndUntagged(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	seedImage(t, reg, repo, "v1")
 	seedImage(t, reg, repo, "v2")
@@ -176,7 +176,7 @@ func TestListImages_KVDigestForm(t *testing.T) {
 }
 
 func TestDeleteImage_ByTagAndByDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	_, digest := seedImage(t, reg, repo, "v1")
 	seedImage(t, reg, repo, "v2") // shares nothing; distinct digest
@@ -205,7 +205,7 @@ func TestDeleteImage_ByTagAndByDigest(t *testing.T) {
 }
 
 func TestDeleteImage_ByDigestClearsTags(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	_, digest := seedImage(t, reg, repo, "v1")
 	// Add a second tag pointing at the same digest.
@@ -221,7 +221,7 @@ func TestDeleteImage_ByDigestClearsTags(t *testing.T) {
 }
 
 func TestStoreManifest_ImmutableTag(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	manA := fmt.Appendf(nil, `{"schemaVersion":2,"mediaType":"%s","layers":[{"digest":"%s"}]}`,
 		mediaTypeDockerManifest, pushBlob(t, reg, repo, []byte("layer-a")))
@@ -254,7 +254,7 @@ func TestStoreManifest_ImmutableTag(t *testing.T) {
 }
 
 func TestStoreManifest_BadDigestReference(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfgDg := pushBlob(t, reg, repo, []byte("cfg"))
 	manifest := fmt.Appendf(nil,

spinifex/domains/ecr/registry/registry_referrer_test.go

@@ -38,7 +38,7 @@ func storedMeta(t *testing.T, reg *Registry, body string) (subject, artifactType
 }
 
 func TestPutManifest_SubjectRecordedAndAdvertised(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 
 	tests := []struct {
@@ -65,7 +65,7 @@ func TestPutManifest_SubjectRecordedAndAdvertised(t *testing.T) {
 }
 
 func TestPutManifest_IndexSubject(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"manifests":[],"subject":{"mediaType":%q,"digest":%q,"size":10}}`,
@@ -81,7 +81,7 @@ func TestPutManifest_IndexSubject(t *testing.T) {
 }
 
 func TestPutManifest_NoSubjectLeavesFieldsEmpty(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"artifactType":%q,"config":{"mediaType":%q,"digest":%q,"size":3},"layers":[]}`,
 		mediaTypeOCIManifest, sigArtifactType, sigConfigType, cfg)
@@ -97,7 +97,7 @@ func TestPutManifest_NoSubjectLeavesFieldsEmpty(t *testing.T) {
 }
 
 func TestPutManifest_MalformedSubjectDigestRejected(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := imageWithSubject(cfg, "sha256:nothex", "")
 
@@ -111,7 +111,7 @@ func TestPutManifest_MalformedSubjectDigestRejected(t *testing.T) {
 }
 
 func TestPutManifest_IndexWithContentTypeParameterGetsNoConfigArtifactType(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"manifests":[],"subject":{"mediaType":%q,"digest":%q,"size":10}}`,
@@ -128,7 +128,7 @@ func TestPutManifest_IndexWithContentTypeParameterGetsNoConfigArtifactType(t *te
 }
 
 func TestDeleteReferrerManifest_RemovesItFromTheSubjectListing(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := imageWithSubject(cfg, absentSubjectDgt, sigArtifactType)
 	dg := digestOf([]byte(body))
@@ -183,7 +183,7 @@ func getReferrersBody(t *testing.T, reg *Registry, path string) (*httptest.Respo
 }
 
 func TestGetReferrers_ListsReferrersAsAnOCIIndex(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dgA := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	dgB := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, "application/vnd.example.sbom.v1", "cfgB")
 
@@ -202,7 +202,7 @@ func TestGetReferrers_ListsReferrersAsAnOCIIndex(t *testing.T) {
 }
 
 func TestGetReferrers_NoReferrersIsAnEmptyIndexNotAnError(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	w := do(reg, http.MethodGet, "/v2/"+referrerRepo+"/referrers/"+absentSubjectDgt, nil, nil)
@@ -212,7 +212,7 @@ func TestGetReferrers_NoReferrersIsAnEmptyIndexNotAnError(t *testing.T) {
 }
 
 func TestGetReferrers_ArtifactTypeFilter(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dgSig := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	pushReferrer(t, reg, referrerRepo, absentSubjectDgt, "application/vnd.example.sbom.v1", "cfgB")
 
@@ -230,7 +230,7 @@ func TestGetReferrers_ArtifactTypeFilter(t *testing.T) {
 }
 
 func TestGetReferrers_MalformedDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	w := do(reg, http.MethodGet, "/v2/"+referrerRepo+"/referrers/not-a-digest", nil, nil)
@@ -239,14 +239,14 @@ func TestGetReferrers_MalformedDigest(t *testing.T) {
 }
 
 func TestGetReferrers_UnknownRepository(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/ghost/referrers/"+absentSubjectDgt, nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "NAME_UNKNOWN")
 }
 
 func TestGetReferrers_DoesNotCrossRepositories(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	seedRepo(t, reg, "team/other")
 
@@ -256,7 +256,7 @@ func TestGetReferrers_DoesNotCrossRepositories(t *testing.T) {
 }
 
 func TestReferrersRoute_OnlyGET(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 	for _, method := range []string{http.MethodPut, http.MethodDelete, http.MethodPost, http.MethodPatch} {
 		w := do(reg, method, "/v2/"+referrerRepo+"/referrers/"+absentSubjectDgt, []byte("{}"), nil)
@@ -272,7 +272,7 @@ func (failingReferrerStore) ListReferrers(context.Context, string, string, strin
 }
 
 func TestGetReferrers_MetadataFailureIsA5xxNotA404(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 	reg.Meta = failingReferrerStore{reg.Meta}
 
@@ -310,7 +310,7 @@ func referrerDigests(t *testing.T, reg *Registry, repo, subject string) []string
 }
 
 func TestDeleteManifest_SubjectDeleteRemovesItsReferrers(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	sbom := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "sbom")
@@ -326,7 +326,7 @@ func TestDeleteManifest_SubjectDeleteRemovesItsReferrers(t *testing.T) {
 }
 
 func TestDeleteManifest_NestedReferrersAreRemoved(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	sigOfSig := pushReferrer(t, reg, referrerRepo, sig, sigArtifactType, "sig-of-sig")
@@ -341,7 +341,7 @@ func TestDeleteManifest_NestedReferrersAreRemoved(t *testing.T) {
 }
 
 func TestDeleteManifest_ReferrerDeleteLeavesTheSubject(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	other := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "sbom")
@@ -355,7 +355,7 @@ func TestDeleteManifest_ReferrerDeleteLeavesTheSubject(t *testing.T) {
 }
 
 func TestDeleteManifest_TagDeleteDoesNotCascade(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	require.NoError(t, reg.Meta.PutTag(context.Background(), testAccount, referrerRepo, "v1", subject))
@@ -368,7 +368,7 @@ func TestDeleteManifest_TagDeleteDoesNotCascade(t *testing.T) {
 }
 
 func TestDeleteManifest_CascadeStaysInsideTheRepository(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	elsewhere := pushReferrer(t, reg, "team/other", subject, sigArtifactType, "sig-other")
@@ -394,7 +394,7 @@ func (s failingDeleteStore) DeleteManifestMeta(ctx context.Context, account, rep
 }
 
 func TestDeleteManifest_ReferrerFailureDoesNotFailTheSubjectDelete(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	stuck := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "stuck")
 	fine := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "fine")

spinifex/domains/ecr/registry/registry_test.go

@@ -2,6 +2,7 @@ package ecrregistry
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/internal/testkit"
 
 	"crypto/sha256"
 	"encoding/hex"
@@ -21,8 +22,15 @@ import (
 
 const testAccount = "000000000000"
 
-func newTestRegistry() *Registry {
-	return NewRegistry(objectstore.NewMemoryObjectStore(), ecr.NewMemoryMetaStore(), testAccount)
+func newTestRegistry(t *testing.T) *Registry {
+	t.Helper()
+	return NewRegistry(objectstore.NewMemoryObjectStore(), newTestMetaStore(t), testAccount)
+}
+
+func newTestMetaStore(t *testing.T) *ecr.KVMetaStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	return ecr.NewKVMetaStore(js)
 }
 
 func digestOf(b []byte) string {
@@ -69,7 +77,7 @@ func pushBlob(t *testing.T, reg *Registry, repo string, data []byte) string {
 }
 
 func TestBlobUpload_Monolithic_And_Head_Get(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("hello layer bytes")
 	dg := pushBlob(t, reg, "team/app", data)
 
@@ -84,20 +92,20 @@ func TestBlobUpload_Monolithic_And_Head_Get(t *testing.T) {
 }
 
 func TestBlobHead_NotFound(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodHead, "/v2/team/app/blobs/"+digestOf([]byte("absent")), nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }
 
 func TestBlobGet_MalformedDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/blobs/sha256:zzz", nil, nil)
 	assert.Equal(t, http.StatusBadRequest, w.Code)
 	assertCode(t, w, "DIGEST_INVALID")
 }
 
 func TestBlobUpload_Chunked(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	c1 := []byte("first-chunk-")
 	c2 := []byte("second-chunk")
@@ -124,7 +132,7 @@ func TestBlobUpload_Chunked(t *testing.T) {
 }
 
 func TestBlobUpload_DigestMismatch(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	w := do(reg, http.MethodPost, "/v2/r/repo/blobs/uploads/", nil, nil)
 	loc := w.Header().Get("Location")
@@ -134,7 +142,7 @@ func TestBlobUpload_DigestMismatch(t *testing.T) {
 }
 
 func TestBlobUpload_Cancel(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	w := do(reg, http.MethodPost, "/v2/r/repo/blobs/uploads/", nil, nil)
 	loc := w.Header().Get("Location")
@@ -161,7 +169,7 @@ func putManifestReferencing(t *testing.T, reg *Registry, repo, digest string) {
 }
 
 func TestBlobMount_HitAndMiss(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("shared-layer")
 	dg := pushBlob(t, reg, "team/source", data)
 	putManifestReferencing(t, reg, "team/source", dg)
@@ -185,7 +193,7 @@ func TestBlobMount_HitAndMiss(t *testing.T) {
 // as a plain upload start, matching the authorization middleware's
 // classification of the same request.
 func TestBlobMount_MissingFromFallsThroughToUpload(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("shared-layer")
 	dg := pushBlob(t, reg, "team/source", data)
 	putManifestReferencing(t, reg, "team/source", dg)
@@ -200,7 +208,7 @@ func TestBlobMount_MissingFromFallsThroughToUpload(t *testing.T) {
 // named source repository is rejected as a mount — proving digest existence
 // alone is not enough to mount cross-repo.
 func TestBlobMount_UnreferencedDigestFallsThrough(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	unreferenced := pushBlob(t, reg, "team/other", []byte("not-in-source"))
 	seedRepo(t, reg, "team/source")
 	seedRepo(t, reg, "team/dest")
@@ -214,7 +222,7 @@ func TestBlobMount_UnreferencedDigestFallsThrough(t *testing.T) {
 // repository that does not exist never mounts, and never distinguishes
 // itself from an unreferenced-digest miss (both fall through identically).
 func TestBlobMount_NonexistentSourceFallsThrough(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dg := pushBlob(t, reg, "team/elsewhere", []byte("some-bytes"))
 	seedRepo(t, reg, "team/dest")
 
@@ -224,7 +232,7 @@ func TestBlobMount_NonexistentSourceFallsThrough(t *testing.T) {
 }
 
 func TestManifest_PutGetHead_ImageManifest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 
 	cfg := []byte(`{"config":true}`)
@@ -255,7 +263,7 @@ func TestManifest_PutGetHead_ImageManifest(t *testing.T) {
 }
 
 func TestManifest_Put_MissingBlob(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	seedRepo(t, reg, repo)
 	manifest := fmt.Appendf(nil,
@@ -268,7 +276,7 @@ func TestManifest_Put_MissingBlob(t *testing.T) {
 }
 
 func TestManifest_Get_AcceptMismatch(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -285,7 +293,7 @@ func TestManifest_Get_AcceptMismatch(t *testing.T) {
 // acceptable manifest media type as its own Accept header line rather than one
 // comma-joined value. Negotiation must consider every line, not just the first.
 func TestManifest_Get_MultipleAcceptHeaders(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -305,7 +313,7 @@ func TestManifest_Get_MultipleAcceptHeaders(t *testing.T) {
 }
 
 func TestManifest_Get_Unknown(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/manifests/missing", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "MANIFEST_UNKNOWN")
@@ -314,7 +322,7 @@ func TestManifest_Get_Unknown(t *testing.T) {
 // TestStartUpload_RepoNotCreated asserts a blob upload to a repository that was
 // never created is rejected up front — ECR does not auto-create on push.
 func TestStartUpload_RepoNotCreated(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodPost, "/v2/team/ghost/blobs/uploads/", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "NAME_UNKNOWN")
@@ -323,7 +331,7 @@ func TestStartUpload_RepoNotCreated(t *testing.T) {
 // TestPutManifest_RepoNotCreated asserts a manifest PUT to an uncreated
 // repository is rejected with NAME_UNKNOWN.
 func TestPutManifest_RepoNotCreated(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	manifest := fmt.Appendf(nil, `{"mediaType":"%s","layers":[]}`, mediaTypeDockerManifest)
 	w := do(reg, http.MethodPut, "/v2/team/ghost/manifests/v1", manifest,
 		map[string]string{"Content-Type": mediaTypeDockerManifest})
@@ -332,7 +340,7 @@ func TestPutManifest_RepoNotCreated(t *testing.T) {
 }
 
 func TestManifest_ImageIndex_ChildValidation(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 
 	// Push a child image manifest first.
@@ -372,7 +380,7 @@ func TestManifest_ImageIndex_ChildValidation(t *testing.T) {
 }
 
 func TestManifest_Delete_Tag(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -390,7 +398,7 @@ func TestManifest_Delete_Tag(t *testing.T) {
 // TestManifest_Delete_ByDigest proves delete-by-digest is no longer a no-op: it
 // removes the manifest record + object, and a subsequent GET is a 404.
 func TestManifest_Delete_ByDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("cfg-bytes"))
 	manifest := fmt.Appendf(nil,
@@ -412,7 +420,7 @@ func TestManifest_Delete_ByDigest(t *testing.T) {
 }
 
 func TestTagsList(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -434,13 +442,13 @@ func TestTagsList(t *testing.T) {
 }
 
 func TestTagsList_UnknownRepo(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/ghost/tags/list", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }
 
 func TestCatalog(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	pushBlob(t, reg, "team/a", []byte("a"))
 	pushBlob(t, reg, "team/b", []byte("b"))
 
@@ -454,21 +462,21 @@ func TestCatalog(t *testing.T) {
 }
 
 func TestDispatch_InvalidRepoName(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/Team/App/tags/list", nil, nil)
 	assert.Equal(t, http.StatusBadRequest, w.Code)
 	assertCode(t, w, "NAME_INVALID")
 }
 
 func TestDispatch_RepeatedSeparatorRepoName(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dg := pushBlob(t, reg, "team/a__b--c", []byte("layer"))
 	w := do(reg, http.MethodHead, "/v2/team/a__b--c/blobs/"+dg, nil, nil)
 	assert.Equal(t, http.StatusOK, w.Code)
 }
 
 func TestDispatch_UnknownPath(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/frobnicate", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }

spinifex/domains/ecr/registry/registry_upload_test.go

@@ -35,7 +35,7 @@ func (s *blobPutFailStore) PutObject(ctx context.Context, input *s3.PutObjectInp
 func TestFinishUpload_StoreFailure_CleansUp(t *testing.T) {
 	mem := objectstore.NewMemoryObjectStore()
 	store := &blobPutFailStore{MemoryObjectStore: mem}
-	meta := ecr.NewMemoryMetaStore()
+	meta := newTestMetaStore(t)
 	reg := NewRegistry(store, meta, testAccount)
 	seedRepo(t, reg, "team/app")
 
@@ -72,7 +72,7 @@ func awsString(s string) *string { return &s }
 // updates issued from the same observed revision cannot both commit. One wins,
 // the other gets ErrConflict — so a racing PATCH is rejected, never merged.
 func TestUploadCAS_SameRevisionSingleWinner(t *testing.T) {
-	meta := ecr.NewMemoryMetaStore()
+	meta := newTestMetaStore(t)
 	rev, err := meta.PutUpload(context.Background(), testAccount, "u-race", ecr.UploadState{RepoName: "r"})
 	require.NoError(t, err)
 
@@ -116,7 +116,7 @@ func TestUploadCAS_SameRevisionSingleWinner(t *testing.T) {
 // hash state and the recorded byte object never desynchronise.
 func TestPatchUpload_ConcurrentNoCorruption(t *testing.T) {
 	for iter := range 25 {
-		reg := newTestRegistry()
+		reg := newTestRegistry(t)
 		repo := "team/race"
 		seedRepo(t, reg, repo)
 

spinifex/domains/ecr/service_impl.go

@@ -16,8 +16,7 @@ type MetaServiceImpl struct {
 
 var _ MetaService = (*MetaServiceImpl)(nil)
 
-// NewMetaServiceImpl builds a MetaService over an explicit MetaStore (used by
-// daemon-side tests with a MemoryMetaStore).
+// NewMetaServiceImpl builds a MetaService over an explicit MetaStore.
 func NewMetaServiceImpl(store MetaStore) *MetaServiceImpl {
 	return &MetaServiceImpl{store: store}
 }

spinifex/domains/ecr/service_test.go

@@ -21,7 +21,7 @@ const svcTestAccount = "000000000000"
 // ErrNotFound/ErrConflict into the response flags rather than transport errors,
 // so they survive the AWS-error-code-only NATS reply.
 func TestMetaServiceImpl_FlagMapping(t *testing.T) {
-	svc := NewMetaServiceImpl(NewMemoryMetaStore())
+	svc := NewMetaServiceImpl(newTestKVMetaStore(t))
 
 	// Absent repo -> Found:false, nil error.
 	dr, err := svc.RepoDescribe(context.Background(), &RepoDescribeRequest{Repo: "ghost"}, svcTestAccount)

spinifex/domains/eks/cluster/delete_cluster_review_test.go

@@ -205,8 +205,6 @@ func TestS2_EKSLastNodegroupDeletedResumesTheClusterDelete(t *testing.T) {
 		ClusterName: aws.String("alpha"), NodegroupName: aws.String("ng1"),
 	}, testAccountID)
 	require.NoError(t, err)
-	_, err = f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
-	require.NoError(t, err)
 	f.svc.WaitTeardowns()
 
 	_, err = GetClusterMeta(t.Context(), f.kv, "alpha")

spinifex/domains/eks/cluster/delete_cluster_test.go

@@ -105,9 +105,6 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture {
 	require.NoError(t, err)
 	// Tight Ready-gating knobs so nodegroup launches resolve in tests without the
 	// production 10m timeout; tests drive meta.NodeCount to simulate the reconciler.
-	svc.noDeleteKick = true
-	svc.workersEndWait = 20 * time.Millisecond
-	svc.workersEndPoll = time.Millisecond
 	svc.nodegroupReadyTimeout = 50 * time.Millisecond
 	svc.nodegroupReadyPoll = 2 * time.Millisecond
 	// Tight worker-launch-retry knobs so a launch-failure retry test resolves

spinifex/domains/eks/cluster/eks_nodegroup_deleting_reaper.go

@@ -1,284 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"strings"
-	"time"
-
-	"github.com/aws/aws-sdk-go/service/eks"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// A DELETING nodegroup becomes DELETE_FAILED once it has been deleting this
-// long and this many passes have failed. Vars for tests.
-var (
-	nodegroupDeleteFailAfter    = 30 * time.Minute
-	nodegroupDeleteFailAttempts = 5
-)
-
-// A pass waits this long, polling, for ended workers to be proven gone.
-const (
-	defaultWorkersEndWait = 90 * time.Second
-	defaultWorkersEndPoll = 3 * time.Second
-)
-
-// errWorkersEnding means workers were told to end but are not yet proven gone.
-var errWorkersEnding = errors.New("workers still ending")
-
-// EKSNodegroupDeletingReaper finishes recorded nodegroup deletes: it ends the
-// workers, removes the nodegroup's own instance profile and only then the
-// record, so a nodegroup is never gone while something it made still is.
-type EKSNodegroupDeletingReaper struct {
-	svc *EKSServiceImpl
-}
-
-var _ vm.Reaper = (*EKSNodegroupDeletingReaper)(nil)
-
-// NewNodegroupDeletingReaper builds the DELETING nodegroup converger.
-func (s *EKSServiceImpl) NewNodegroupDeletingReaper() *EKSNodegroupDeletingReaper {
-	return &EKSNodegroupDeletingReaper{svc: s}
-}
-
-func (r *EKSNodegroupDeletingReaper) Class() string         { return "eks-nodegroup-deleting" }
-func (r *EKSNodegroupDeletingReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
-
-// Sweep hands every due DELETING nodegroup to a background converger and
-// returns how many it started. The lease decides which node runs each one.
-func (r *EKSNodegroupDeletingReaper) Sweep(ctx context.Context) (int, error) {
-	if !r.svc.depsReadyForOrchestration() {
-		return 0, nil
-	}
-	js, err := jetstream.New(r.svc.deps.NATSConn)
-	if err != nil {
-		return 0, fmt.Errorf("jetstream: %w", err)
-	}
-	names, err := accountBucketNames(ctx, r.svc.deps.NATSConn)
-	if err != nil {
-		return 0, fmt.Errorf("enumerate account buckets: %w", err)
-	}
-
-	started := 0
-	for _, name := range names {
-		select {
-		case <-ctx.Done():
-			return started, ctx.Err()
-		default:
-		}
-		accountID := strings.TrimPrefix(name, KVBucketEKSAccountPrefix)
-		acctKV, err := js.KeyValue(ctx, name)
-		if err != nil {
-			continue
-		}
-		clusters, err := listClusterNames(ctx, acctKV)
-		if err != nil {
-			continue
-		}
-		for _, cluster := range clusters {
-			recs, err := ListNodegroupRecords(ctx, acctKV, cluster)
-			if err != nil {
-				continue
-			}
-			for _, rec := range recs {
-				if r.due(rec) && r.svc.startNodegroupDelete(accountID, acctKV, cluster, rec.Name) {
-					started++
-				}
-			}
-		}
-	}
-	return started, nil
-}
-
-// due reports whether rec is DELETING and past its retry backoff.
-func (r *EKSNodegroupDeletingReaper) due(rec *NodegroupRecord) bool {
-	if rec.Status != eks.NodegroupStatusDeleting {
-		return false
-	}
-	if rec.DeleteReapAttempts == 0 || rec.LastDeleteReapAttempt.IsZero() {
-		return true
-	}
-	return time.Since(rec.LastDeleteReapAttempt) >= deleteReapBackoff(deletingReapMinAge, rec.DeleteReapAttempts)
-}
-
-// startNodegroupDelete runs one converger pass in the background and reports
-// whether it started; a pass already running here or on another node refuses.
-func (s *EKSServiceImpl) startNodegroupDelete(accountID string, kv jetstream.KeyValue, cluster, ng string) bool {
-	release, _, ok := s.holdTeardown(accountID, cluster+"/nodegroup/"+ng, false)
-	if !ok {
-		return false
-	}
-	s.teardownWG.Go(func() {
-		defer release()
-		ctx, cancel := context.WithTimeout(s.bgCtx, teardownPassTimeout)
-		defer cancel()
-		if err := s.convergeNodegroupDeletion(ctx, accountID, kv, cluster, ng); err != nil {
-			slog.Warn("eks: nodegroup delete incomplete, the record stays DELETING for retry",
-				"cluster", cluster, "nodegroup", ng, "account", accountID, "err", err)
-		}
-	})
-	return true
-}
-
-// WaitNodegroupDeletes blocks until every converger this process started has
-// returned. Intended for tests.
-func (s *EKSServiceImpl) WaitNodegroupDeletes() { s.teardownWG.Wait() }
-
-// mutateNodegroup applies fn to the stored record under compare-and-swap. fn
-// returns false to leave the record unchanged.
-func (s *EKSServiceImpl) mutateNodegroup(ctx context.Context, kv jetstream.KeyValue, cluster, ng string, fn func(*NodegroupRecord) bool) (*NodegroupRecord, error) {
-	for range ngCASMaxRetries {
-		rec, rev, err := getNodegroupEntry(ctx, kv, cluster, ng)
-		if err != nil {
-			return nil, err
-		}
-		if !fn(rec) {
-			return rec, nil
-		}
-		ok, err := s.casPutNodegroup(ctx, kv, rec, rev)
-		if err != nil {
-			return nil, err
-		}
-		if ok {
-			return rec, nil
-		}
-	}
-	return nil, fmt.Errorf("nodegroup %s/%s: concurrent writers kept winning", cluster, ng)
-}
-
-// convergeNodegroupDeletion finishes one recorded delete. It acts only on a
-// record that is DELETING when re-read, and keeps the record on any failure.
-func (s *EKSServiceImpl) convergeNodegroupDeletion(ctx context.Context, accountID string, kv jetstream.KeyValue, cluster, ng string) error {
-	rec, err := s.mutateNodegroup(ctx, kv, cluster, ng, func(r *NodegroupRecord) bool {
-		if r.Status != eks.NodegroupStatusDeleting {
-			return false
-		}
-		r.DeleteReapAttempts++
-		r.LastDeleteReapAttempt = time.Now().UTC()
-		return true
-	})
-	if err != nil {
-		if errors.Is(err, ErrNodegroupNotFound) {
-			return nil
-		}
-		return err
-	}
-	if rec.Status != eks.NodegroupStatusDeleting {
-		return nil
-	}
-
-	if err := s.releaseNodegroupResources(ctx, accountID, kv, rec); err != nil {
-		if errors.Is(err, errWorkersEnding) {
-			s.noteWorkersEnding(ctx, kv, cluster, ng, err)
-			return err
-		}
-		s.noteNodegroupDeleteFailure(ctx, kv, rec, err)
-		return err
-	}
-	if err := DeleteNodegroupRecord(ctx, kv, cluster, ng); err != nil {
-		return err
-	}
-	// A cluster delete waits on its node groups; the last one going lets it run.
-	if meta, err := GetClusterMeta(ctx, kv, cluster); err == nil && meta.Status == ClusterStatusDeleting {
-		if rest, err := ListNodegroupRecords(ctx, kv, cluster); err == nil && len(rest) == 0 {
-			s.startTeardown(accountID, cluster)
-		}
-	}
-	return nil
-}
-
-// releaseNodegroupResources ends the workers, then removes the profile. The
-// profile and the recorded worker ids go only once every worker is proven
-// terminated or absent, since an accepted terminate is only a request.
-func (s *EKSServiceImpl) releaseNodegroupResources(ctx context.Context, accountID string, kv jetstream.KeyValue, rec *NodegroupRecord) error {
-	if len(rec.InstanceIDs) > 0 {
-		if err := s.deps.Worker.TerminateWorkerInstances(ctx, rec.InstanceIDs, accountID); err != nil {
-			return fmt.Errorf("terminate workers: %w", err)
-		}
-		if err := s.awaitWorkersGone(ctx, rec.InstanceIDs, accountID); err != nil {
-			return err
-		}
-		if _, err := s.mutateNodegroup(ctx, kv, rec.ClusterName, rec.Name, func(r *NodegroupRecord) bool {
-			r.InstanceIDs = nil
-			return true
-		}); err != nil {
-			return fmt.Errorf("record workers ended: %w", err)
-		}
-	}
-	return s.deleteNodegroupProfile(ctx, accountID, kv, rec)
-}
-
-// awaitWorkersGone polls until no worker is live, up to the end wait,
-// and returns errWorkersEnding if some still are.
-func (s *EKSServiceImpl) awaitWorkersGone(ctx context.Context, ids []string, accountID string) error {
-	wait, poll := s.workersEndWait, s.workersEndPoll
-	if wait <= 0 {
-		wait = defaultWorkersEndWait
-	}
-	if poll <= 0 {
-		poll = defaultWorkersEndPoll
-	}
-	deadline := time.Now().Add(wait)
-	for {
-		live, err := s.deps.Worker.LiveWorkerInstances(ctx, ids, accountID)
-		if err != nil {
-			return fmt.Errorf("confirm workers ended: %w", err)
-		}
-		if len(live) == 0 {
-			return nil
-		}
-		if !time.Now().Before(deadline) {
-			return fmt.Errorf("%w: %s", errWorkersEnding, strings.Join(live, ","))
-		}
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-time.After(poll):
-		}
-	}
-}
-
-// noteWorkersEnding handles workers that are not yet gone. Early on that is no
-// failed attempt: the next sweep checks again without backoff. Once the group
-// has been deleting past nodegroupDeleteFailAfter it settles DELETE_FAILED
-// naming the instances, keeping the profile and record for a customer retry.
-func (s *EKSServiceImpl) noteWorkersEnding(ctx context.Context, kv jetstream.KeyValue, cluster, ng string, cause error) {
-	_, err := s.mutateNodegroup(ctx, kv, cluster, ng, func(r *NodegroupRecord) bool {
-		if r.Status != eks.NodegroupStatusDeleting {
-			return false
-		}
-		if time.Since(r.DeletingSince) >= nodegroupDeleteFailAfter {
-			r.Status = eks.NodegroupStatusDeleteFailed
-			r.StatusReason = "delete: " + cause.Error()
-			r.ModifiedAt = time.Now().UTC()
-			return true
-		}
-		r.DeleteReapAttempts--
-		r.LastDeleteReapAttempt = time.Time{}
-		return true
-	})
-	if err != nil {
-		slog.WarnContext(ctx, "eks: record workers still ending", "cluster", cluster, "nodegroup", ng, "err", err)
-	}
-}
-
-// noteNodegroupDeleteFailure records why a pass failed and, once the group has
-// been deleting long enough and failed often enough, settles it DELETE_FAILED.
-func (s *EKSServiceImpl) noteNodegroupDeleteFailure(ctx context.Context, kv jetstream.KeyValue, rec *NodegroupRecord, cause error) {
-	_, err := s.mutateNodegroup(ctx, kv, rec.ClusterName, rec.Name, func(r *NodegroupRecord) bool {
-		if r.Status != eks.NodegroupStatusDeleting {
-			return false
-		}
-		r.StatusReason = "delete: " + cause.Error()
-		r.ModifiedAt = time.Now().UTC()
-		if r.DeleteReapAttempts >= nodegroupDeleteFailAttempts && time.Since(r.DeletingSince) >= nodegroupDeleteFailAfter {
-			r.Status = eks.NodegroupStatusDeleteFailed
-		}
-		return true
-	})
-	if err != nil {
-		slog.WarnContext(ctx, "eks: record nodegroup delete failure", "cluster", rec.ClusterName, "nodegroup", rec.Name, "err", err)
-	}
-}

spinifex/domains/eks/cluster/nodegroup.go

@@ -379,12 +379,6 @@ func (s *EKSServiceImpl) createNodegroup(ctx context.Context, acctKV jetstream.K
 		ModifiedAt:     now,
 	}
 
-	// Recorded before the claim, so a name is durable before any worker launches.
-	if rec.NodeRole != "" {
-		rec.InstanceProfileID = uuid.NewV4().String()
-		rec.InstanceProfileName = nodegroupProfilePrefix + rec.InstanceProfileID
-	}
-
 	// Atomically claim the record; concurrent/duplicate CreateNodegroup requests lose here.
 	owned, err := ClaimNodegroupRecord(ctx, acctKV, rec)
 	if err != nil {
@@ -722,7 +716,7 @@ func (s *EKSServiceImpl) launchOneWorker(ctx context.Context, rec *NodegroupReco
 	// Attach the node role's instance profile so IMDS serves the role to the ECR
 	// credential provider; without it worker pulls from the internal ECR get 401.
 	if s.iamEnsurer() != nil && rec.NodeRole != "" {
-		profileARN, perr := s.nodeProfileForLaunch(accountID, rec)
+		profileARN, perr := s.ensureNodeInstanceProfile(accountID, rec.NodeRole)
 		if perr != nil {
 			return "", fmt.Errorf("ensure node instance profile: %w", perr)
 		}
@@ -861,10 +855,14 @@ func gatewayHostIP(gatewayURL string) string {
 	return host
 }
 
-// resolveNodeRoleName returns the name of the stored role nodeRoleARN names.
-// Resolution compares the supplied ARN against the stored one, so an invented
-// path cannot bind a profile to the role its trailing name happens to match.
-func (s *EKSServiceImpl) resolveNodeRoleName(accountID, nodeRoleARN string) (string, error) {
+// ensureNodeInstanceProfile guarantees an IAM instance profile named for the
+// nodegroup's node role exists and carries that role, returning its ARN. Workers
+// launch with this profile so IMDS serves the node role to the ECR credential
+// provider, mirroring the implicit instance profile real EKS creates for a node
+// role. Idempotent: concurrent worker launches converge on the same profile.
+func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string) (string, error) {
+	// Resolution compares the supplied ARN against the stored one, so an invented
+	// path cannot bind a profile to the role its trailing name happens to match.
 	_, roleName, err := auth.ResolveRoleARN(nodeRoleARN, func(account, name string) (string, error) {
 		// The profile is created under the caller's account, so a role from
 		// another account would silently bind a name that account does not own.
@@ -883,19 +881,6 @@ func (s *EKSServiceImpl) resolveNodeRoleName(accountID, nodeRoleARN string) (str
 	if err != nil {
 		return "", fmt.Errorf("node role ARN %q: %w", nodeRoleARN, err)
 	}
-	return roleName, nil
-}
-
-// ensureNodeInstanceProfile guarantees an IAM instance profile named for the
-// nodegroup's node role exists and carries that role, returning its ARN. Workers
-// launch with this profile so IMDS serves the node role to the ECR credential
-// provider, mirroring the implicit instance profile real EKS creates for a node
-// role. Idempotent: concurrent worker launches converge on the same profile.
-func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string) (string, error) {
-	roleName, err := s.resolveNodeRoleName(accountID, nodeRoleARN)
-	if err != nil {
-		return "", err
-	}
 	profileName := roleName
 
 	out, err := s.iamEnsurer().GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{
@@ -1033,9 +1018,6 @@ func (s *EKSServiceImpl) reconcileNodegroup(ctx context.Context, acctKV jetstrea
 			}
 			return nil, err
 		}
-		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusDeleteFailed {
-			return nil, errors.New(awserrors.ErrorEKSResourceInUse)
-		}
 		applyScalingUpdate(rec, scaling)
 		if labels != nil {
 			rec.Labels = applyLabelUpdate(rec.Labels, labels)
@@ -1245,47 +1227,31 @@ func (s *EKSServiceImpl) deleteNodegroup(ctx context.Context, acctKV jetstream.K
 	if err := requireCluster(ctx, acctKV, cluster); err != nil {
 		return nil, err
 	}
-	rec, err := s.markNodegroupDeleting(ctx, acctKV, cluster, ng)
+	rec, err := GetNodegroupRecord(ctx, acctKV, cluster, ng)
 	if err != nil {
+		if errors.Is(err, ErrNodegroupNotFound) {
+			return nil, nodegroupNotFound(ng)
+		}
 		return nil, err
 	}
-	if !s.noDeleteKick {
-		s.startNodegroupDelete(accountID, acctKV, cluster, ng)
-	}
-	return &eks.DeleteNodegroupOutput{Nodegroup: nodegroupRecordToAWS(rec)}, nil
-}
 
-// markNodegroupDeleting records DELETING and returns. The workers, the profile
-// and the record itself are the nodegroup reaper's to remove, so the request
-// never waits on work that scales with the group (a launching group is refused).
-func (s *EKSServiceImpl) markNodegroupDeleting(ctx context.Context, kv jetstream.KeyValue, cluster, ng string) (*NodegroupRecord, error) {
-	for range ngCASMaxRetries {
-		rec, rev, err := getNodegroupEntry(ctx, kv, cluster, ng)
-		if err != nil {
-			if errors.Is(err, ErrNodegroupNotFound) {
-				return nil, nodegroupNotFound(ng)
-			}
-			return nil, err
-		}
-		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusCreating {
-			return nil, errors.New(awserrors.ErrorEKSResourceInUse)
-		}
-		now := time.Now().UTC()
-		rec.Status = eks.NodegroupStatusDeleting
-		rec.StatusReason = ""
-		rec.DeletingSince = now
-		rec.DeleteReapAttempts = 0
-		rec.LastDeleteReapAttempt = time.Time{}
-		rec.ModifiedAt = now
-		ok, err := s.casPutNodegroup(ctx, kv, rec, rev)
-		if err != nil {
-			return nil, err
-		}
-		if ok {
-			return rec, nil
+	if len(rec.InstanceIDs) > 0 {
+		if err := s.deps.Worker.TerminateWorkerInstances(ctx, rec.InstanceIDs, accountID); err != nil {
+			return nil, fmt.Errorf("terminate workers: %w", err)
 		}
 	}
-	return nil, errors.New(awserrors.ErrorServerInternal)
+	if err := DeleteNodegroupRecord(ctx, acctKV, cluster, ng); err != nil {
+		return nil, err
+	}
+	// A cluster delete waits on its node groups; the last one going lets it run.
+	if meta, err := GetClusterMeta(ctx, acctKV, cluster); err == nil && meta.Status == ClusterStatusDeleting {
+		if rest, err := ListNodegroupRecords(ctx, acctKV, cluster); err == nil && len(rest) == 0 {
+			s.startTeardown(accountID, cluster)
+		}
+	}
+
+	rec.Status = eks.NodegroupStatusDeleting
+	return &eks.DeleteNodegroupOutput{Nodegroup: nodegroupRecordToAWS(rec)}, nil
 }
 
 // decryptNodeToken reads + decrypts the cluster's K3s join token from KV.
@@ -1530,12 +1496,8 @@ func nodegroupRecordToAWS(rec *NodegroupRecord) *eks.Nodegroup {
 	// wrong; a nil slice here would marshal the key away entirely.
 	issues := make([]*eks.Issue, 0, 1)
 	if rec.StatusReason != "" {
-		code := eks.NodegroupIssueCodeNodeCreationFailure
-		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusDeleteFailed {
-			code = eks.NodegroupIssueCodeNodeTerminationFailure
-		}
 		issues = append(issues, &eks.Issue{
-			Code:        aws.String(code),
+			Code:        aws.String(eks.NodegroupIssueCodeNodeCreationFailure),
 			Message:     aws.String(rec.StatusReason),
 			ResourceIds: aws.StringSlice(rec.InstanceIDs),
 		})

spinifex/domains/eks/cluster/nodegroup_delete_failure_test.go

@@ -1,255 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"errors"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/service/eks"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func (f *fakeWorkerLauncher) terminateCallCount() int {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return len(f.terminateCalls)
-}
-
-func requireNodegroupStatus(t *testing.T, f *eksServiceFixture, cluster, ng, want string) *NodegroupRecord {
-	t.Helper()
-	rec, err := GetNodegroupRecord(t.Context(), f.kv, cluster, ng)
-	require.NoError(t, err)
-	require.Equal(t, want, rec.Status)
-	return rec
-}
-
-func TestNodegroupDeletingReaper_SweepWithoutOrchestrationDepsStartsNothing(t *testing.T) {
-	t.Parallel()
-	n, err := (&EKSServiceImpl{}).NewNodegroupDeletingReaper().Sweep(t.Context())
-	require.NoError(t, err)
-	assert.Zero(t, n)
-}
-
-func TestNodegroupDeletingReaper_CancelledSweepLeavesTheDeleteRecorded(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	ctx, cancel := context.WithCancel(t.Context())
-	cancel()
-	n, err := f.svc.NewNodegroupDeletingReaper().Sweep(ctx)
-	f.svc.WaitNodegroupDeletes()
-
-	require.ErrorIs(t, err, context.Canceled)
-	assert.Zero(t, n)
-	requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusDeleting)
-}
-
-func TestNodegroupDeletingReaper_OnlyDeletingGroupsPastBackoffAreStarted(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "active", 1)
-	createReadyNodegroup(t, f, "c1", "backoff", 1)
-	_, err := deleteNodegroupCall(f, "c1", "backoff")
-	require.NoError(t, err)
-	rec := requireNodegroupStatus(t, f, "c1", "backoff", eks.NodegroupStatusDeleting)
-	rec.DeleteReapAttempts = 3
-	rec.LastDeleteReapAttempt = time.Now().UTC()
-	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, rec))
-
-	assert.Zero(t, sweepNodegroups(t, f), "an ACTIVE group and a DELETING one inside its backoff are not due")
-	requireNodegroupStatus(t, f, "c1", "active", eks.NodegroupStatusActive)
-	assert.Zero(t, f.worker.terminateCallCount())
-}
-
-func TestConvergeNodegroupDeletion_MissingOrNotDeletingRecordIsLeftAlone(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-
-	require.NoError(t, f.svc.convergeNodegroupDeletion(t.Context(), testAccountID, f.kv, "c1", "never-existed"))
-	require.NoError(t, f.svc.convergeNodegroupDeletion(t.Context(), testAccountID, f.kv, "c1", "ng1"))
-
-	requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusActive)
-	assert.Zero(t, f.worker.terminateCallCount(), "a group nobody asked to delete keeps its workers")
-	assert.Contains(t, fixtureIAM(t, f).Profiles, mustProfileName(t, f, "c1", "ng1"))
-}
-
-func mustProfileName(t *testing.T, f *eksServiceFixture, cluster, ng string) string {
-	t.Helper()
-	rec, _ := ownedProfile(t, f, cluster, ng)
-	return rec.InstanceProfileName
-}
-
-func TestAwaitWorkersGone_CancelledWhileWorkersEndingReturnsTheContextError(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	f.worker.stillLive = []string{"i-slow"}
-	f.svc.workersEndWait = time.Hour
-	f.svc.workersEndPoll = time.Hour
-	ctx, cancel := context.WithCancel(t.Context())
-	time.AfterFunc(50*time.Millisecond, cancel)
-
-	err := f.svc.awaitWorkersGone(ctx, []string{"i-slow"}, testAccountID)
-
-	require.ErrorIs(t, err, context.Canceled)
-	assert.NotErrorIs(t, err, errWorkersEnding, "a cancelled wait is not a verdict on the workers")
-}
-
-func TestDeleteNodegroup_IAMReadFailureKeepsGroupDeletingAndProfileIntact(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-	m := fixtureIAM(t, f)
-	m.GetInstanceProfileErr = errors.New("iam read timed out")
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	sweepNodegroups(t, f)
-
-	cur := requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusDeleting)
-	assert.Contains(t, cur.StatusReason, "get instance profile", "the failed read is recorded for the customer")
-	assert.Contains(t, m.Profiles, rec.InstanceProfileName, "an unreadable profile is not a missing one")
-	assert.Zero(t, m.DeleteInstanceProfileCalls)
-
-	m.GetInstanceProfileErr = nil
-	backdateNodegroupReap(t, f, "c1", "ng1")
-	sweepNodegroups(t, f)
-	_, err = describeNodegroupCall(f, "c1", "ng1")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
-}
-
-func TestDeleteNodegroup_ProfileAlreadyGoneStillFinishesTheGroup(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-	m := fixtureIAM(t, f)
-	delete(m.Profiles, rec.InstanceProfileName)
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	sweepNodegroups(t, f)
-
-	_, err = describeNodegroupCall(f, "c1", "ng1")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-	assert.Zero(t, m.RemoveRoleCalls)
-	assert.Zero(t, m.DeleteInstanceProfileCalls)
-}
-
-func TestDeleteNodegroup_ProfileVanishingMidDeleteStillFinishesTheGroup(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	m := fixtureIAM(t, f)
-	m.DeleteInstanceProfileErr = awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile gone.")
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	sweepNodegroups(t, f)
-
-	_, err = describeNodegroupCall(f, "c1", "ng1")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-	assert.Equal(t, 1, m.RemoveRoleCalls)
-}
-
-func TestDeleteNodegroup_DeleteProfileFailureKeepsGroupDeletingUntilItClears(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-	m := fixtureIAM(t, f)
-	m.DeleteInstanceProfileErr = errors.New("iam unavailable")
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	sweepNodegroups(t, f)
-	cur := requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusDeleting)
-	assert.Contains(t, cur.StatusReason, "delete instance profile")
-	assert.Empty(t, cur.InstanceIDs, "workers proven gone are not terminated again")
-
-	m.DeleteInstanceProfileErr = nil
-	backdateNodegroupReap(t, f, "c1", "ng1")
-	sweepNodegroups(t, f)
-	_, err = describeNodegroupCall(f, "c1", "ng1")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
-}
-
-func TestEnsureNodegroupProfile_RecreatesAMissingRecordedProfileAsItsOwn(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-	m := fixtureIAM(t, f)
-	delete(m.Profiles, rec.InstanceProfileName)
-
-	arn, err := f.svc.ensureNodegroupProfile(testAccountID, rec)
-
-	require.NoError(t, err)
-	assert.Contains(t, arn, rec.InstanceProfileName)
-	got := m.Profiles[rec.InstanceProfileName]
-	require.NotNil(t, got)
-	assert.True(t, profileOwnedBy(got, rec), "a re-created profile carries this incarnation's tags")
-	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(got))
-}
-
-func TestEnsureNodegroupProfile_RefusesAProfileThatIsNotThisGroups(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	rec, p := ownedProfile(t, f, "c1", "ng1")
-	p.Tags = nil
-	callsBefore := fixtureIAM(t, f).AddRoleToInstanceProfileCalls
-
-	_, err := f.svc.ensureNodegroupProfile(testAccountID, rec)
-
-	require.ErrorContains(t, err, "is not this nodegroup's")
-	assert.Equal(t, callsBefore, fixtureIAM(t, f).AddRoleToInstanceProfileCalls, "no role is attached to a profile EKS does not own")
-}
-
-func TestEnsureNodegroupProfile_IAMFailuresSurfaceWithoutHalfWork(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-	m := fixtureIAM(t, f)
-
-	m.GetInstanceProfileErr = errors.New("iam read failed")
-	_, err := f.svc.ensureNodegroupProfile(testAccountID, rec)
-	require.ErrorContains(t, err, "get instance profile")
-	m.GetInstanceProfileErr = nil
-
-	delete(m.Profiles, rec.InstanceProfileName)
-	m.CreateInstanceProfileErr = errors.New("iam write failed")
-	_, err = f.svc.ensureNodegroupProfile(testAccountID, rec)
-	require.ErrorContains(t, err, "create instance profile")
-	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
-}
-
-func TestMarkNodegroupDeleting_UnknownGroupIsResourceNotFound(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-
-	_, err := f.svc.markNodegroupDeleting(t.Context(), f.kv, "c1", "ghost")
-
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-}

spinifex/domains/eks/cluster/nodegroup_delete_test.go

@@ -1,662 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"errors"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/aws/aws-sdk-go/service/eks"
-	"github.com/aws/aws-sdk-go/service/iam"
-	iammock "github.com/mulgadc/spinifex/spinifex/domains/iam/mock"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	profileClusterTag   = "spinifex:eks-cluster"
-	profileNodegroupTag = "spinifex:eks-nodegroup"
-	fixtureNodeRoleName = "eks-node"
-)
-
-func fixtureIAM(t *testing.T, f *eksServiceFixture) *iammock.SystemInstanceRoleEnsurer {
-	t.Helper()
-	m, ok := f.svc.deps.IAM.(*iammock.SystemInstanceRoleEnsurer)
-	require.True(t, ok, "the fixture IAM must be the in-memory mock")
-	return m
-}
-
-// createReadyNodegroup creates a nodegroup whose n workers register Ready.
-func createReadyNodegroup(t *testing.T, f *eksServiceFixture, cluster, ng string, n int64) {
-	t.Helper()
-	markWorkersReady(t, f, cluster, ng, int(n))
-	_, err := f.svc.CreateNodegroup(context.Background(), createNGInput(cluster, ng, n), testAccountID)
-	require.NoError(t, err)
-	f.svc.WaitLaunches()
-}
-
-// createUnreadyNodegroup creates a nodegroup that never registers, so it ends
-// CREATE_FAILED holding the workers it launched.
-func createUnreadyNodegroup(t *testing.T, f *eksServiceFixture, cluster, ng string, n int64) {
-	t.Helper()
-	_, err := f.svc.CreateNodegroup(context.Background(), createNGInput(cluster, ng, n), testAccountID)
-	require.NoError(t, err)
-	f.svc.WaitLaunches()
-}
-
-func deleteNodegroupCall(f *eksServiceFixture, cluster, ng string) (*eks.DeleteNodegroupOutput, error) {
-	return f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
-		ClusterName: aws.String(cluster), NodegroupName: aws.String(ng),
-	}, testAccountID)
-}
-
-func describeNodegroupCall(f *eksServiceFixture, cluster, ng string) (*eks.DescribeNodegroupOutput, error) {
-	return f.svc.DescribeNodegroup(context.Background(), &eks.DescribeNodegroupInput{
-		ClusterName: aws.String(cluster), NodegroupName: aws.String(ng),
-	}, testAccountID)
-}
-
-func sweepNodegroups(t *testing.T, f *eksServiceFixture) int {
-	t.Helper()
-	n, err := f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
-	require.NoError(t, err)
-	f.svc.WaitNodegroupDeletes()
-	return n
-}
-
-// backdateNodegroupReap moves the last reap pass far enough back that any
-// retry backoff has elapsed.
-func backdateNodegroupReap(t *testing.T, f *eksServiceFixture, cluster, ng string) {
-	t.Helper()
-	rec, err := GetNodegroupRecord(t.Context(), f.kv, cluster, ng)
-	require.NoError(t, err)
-	rec.LastDeleteReapAttempt = time.Now().UTC().Add(-24 * time.Hour)
-	rec.DeletingSince = time.Now().UTC().Add(-24 * time.Hour)
-	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, rec))
-}
-
-func requireErrorCode(t *testing.T, err error, want string) {
-	t.Helper()
-	require.Error(t, err)
-	code, ok := awserrors.ResolveErrorCode(err)
-	require.True(t, ok, "error %v carries no AWS code", err)
-	assert.Equal(t, want, code)
-}
-
-func profileTag(p *iam.InstanceProfile, key string) string {
-	for _, tg := range p.Tags {
-		if aws.StringValue(tg.Key) == key {
-			return aws.StringValue(tg.Value)
-		}
-	}
-	return ""
-}
-
-func profileHasTagValue(p *iam.InstanceProfile, value string) bool {
-	for _, tg := range p.Tags {
-		if aws.StringValue(tg.Value) == value {
-			return true
-		}
-	}
-	return false
-}
-
-func profileRoleNames(p *iam.InstanceProfile) []string {
-	var names []string
-	for _, r := range p.Roles {
-		names = append(names, aws.StringValue(r.RoleName))
-	}
-	return names
-}
-
-// ownedProfile returns the nodegroup's record and the instance profile it
-// names, failing the test if either is missing.
-func ownedProfile(t *testing.T, f *eksServiceFixture, cluster, ng string) (*NodegroupRecord, *iam.InstanceProfile) {
-	t.Helper()
-	rec, err := GetNodegroupRecord(t.Context(), f.kv, cluster, ng)
-	require.NoError(t, err)
-	require.NotEmpty(t, rec.InstanceProfileName, "the record must name the profile it owns")
-	require.NotEmpty(t, rec.InstanceProfileID, "the record must carry the per-incarnation id")
-	p, ok := fixtureIAM(t, f).Profiles[rec.InstanceProfileName]
-	require.True(t, ok, "profile %q must exist", rec.InstanceProfileName)
-	return rec, p
-}
-
-func seedCustomerProfile(f *eksServiceFixture, t *testing.T, name, role string, tags map[string]string) {
-	t.Helper()
-	p := &iam.InstanceProfile{
-		InstanceProfileName: aws.String(name),
-		Arn:                 aws.String("arn:aws:iam::" + testAccountID + ":instance-profile/" + name),
-	}
-	if role != "" {
-		p.Roles = []*iam.Role{{RoleName: aws.String(role)}}
-	}
-	for k, v := range tags {
-		p.Tags = append(p.Tags, &iam.Tag{Key: aws.String(k), Value: aws.String(v)})
-	}
-	fixtureIAM(t, f).Profiles[name] = p
-}
-
-func TestDeleteNodegroup_ReturnsDeletingWithoutTerminatingInRequest(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 2)
-
-	out, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-	assert.Equal(t, eks.NodegroupStatusDeleting, aws.StringValue(out.Nodegroup.Status))
-
-	assert.Zero(t, f.worker.terminatedCount(), "the request must not wait for workers to end")
-	rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
-	require.NoError(t, err, "the record stays until the reaper has finished")
-	assert.Equal(t, eks.NodegroupStatusDeleting, rec.Status)
-
-	desc, err := describeNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-	assert.Equal(t, eks.NodegroupStatusDeleting, aws.StringValue(desc.Nodegroup.Status))
-}
-
-func TestDeleteNodegroup_ReaperEndsWorkersThenRemovesGroup(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 2)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-	assert.Equal(t, 1, sweepNodegroups(t, f))
-
-	require.Len(t, f.worker.terminateCalls, 1)
-	assert.ElementsMatch(t, rec.InstanceIDs, f.worker.terminateCalls[0])
-	_, err = describeNodegroupCall(f, "c1", "ng1")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-	_, ok := fixtureIAM(t, f).Profiles[rec.InstanceProfileName]
-	assert.False(t, ok, "the group is not gone until its profile is")
-}
-
-func TestDeleteNodegroup_RepeatedDeleteOnDeletingGroupIsResourceInUse(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	_, err = deleteNodegroupCall(f, "c1", "ng1")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
-}
-
-func TestDeleteNodegroup_DeletingGroupRefusesUpdates(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	_, err = f.svc.UpdateNodegroupConfig(context.Background(), &eks.UpdateNodegroupConfigInput{
-		ClusterName:   aws.String("c1"),
-		NodegroupName: aws.String("ng1"),
-		ScalingConfig: &eks.NodegroupScalingConfig{MinSize: aws.Int64(1), MaxSize: aws.Int64(3), DesiredSize: aws.Int64(3)},
-	}, testAccountID)
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
-	assert.Equal(t, 1, f.worker.runCount(), "a DELETING group must not launch workers")
-}
-
-func TestDeleteNodegroup_SameNameCreateWhileDeletingIsResourceInUse(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	_, err = f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
-}
-
-func TestDeleteNodegroup_DeleteClusterRefusedWhileGroupDeleting(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	_, err = f.svc.DeleteCluster(context.Background(), &eks.DeleteClusterInput{Name: aws.String("c1")}, testAccountID)
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
-}
-
-func TestDeleteNodegroup_BootReclaimLeavesDeletingGroupToReaper(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{
-		ClusterName: "c1", Name: "ng1", Status: eks.NodegroupStatusDeleting,
-		InstanceIDs: []string{"i-w1", "i-w2"}, NodeRole: fixtureNodeRoleARN,
-	}))
-
-	f.svc.reclaimOrphanedNodegroups(t.Context(), testAccountID, f.kv, "c1")
-
-	assert.Zero(t, f.worker.terminatedCount())
-	rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
-	require.NoError(t, err)
-	assert.Equal(t, eks.NodegroupStatusDeleting, rec.Status)
-	assert.Equal(t, []string{"i-w1", "i-w2"}, rec.InstanceIDs)
-}
-
-func TestDeleteNodegroup_RepeatedReaperFailureEndsDeleteFailedWithHealthIssue(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	f.worker.termErr = errors.New("terminate refused")
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	var rec *NodegroupRecord
-	for range 30 {
-		sweepNodegroups(t, f)
-		rec, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
-		require.NoError(t, err, "a failing delete must leave the record, never drop it")
-		if rec.Status == eks.NodegroupStatusDeleteFailed {
-			break
-		}
-		backdateNodegroupReap(t, f, "c1", "ng1")
-	}
-	require.Equal(t, eks.NodegroupStatusDeleteFailed, rec.Status)
-
-	desc, err := describeNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-	require.NotEmpty(t, desc.Nodegroup.Health.Issues, "DELETE_FAILED must say why")
-	assert.Equal(t, eks.NodegroupIssueCodeNodeTerminationFailure, aws.StringValue(desc.Nodegroup.Health.Issues[0].Code))
-
-	// The customer retries the delete once the cause is gone.
-	f.worker.termErr = nil
-	_, err = deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-	sweepNodegroups(t, f)
-	_, err = describeNodegroupCall(f, "c1", "ng1")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-}
-
-func TestCreateNodegroup_OwnsATaggedProfileRecordedBeforeFirstLaunch(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	var seenAtLaunch string
-	f.worker.beforeRun = func(in *ec2.RunInstancesInput) {
-		if rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1"); err == nil && seenAtLaunch == "" {
-			seenAtLaunch = rec.InstanceProfileName
-		}
-	}
-	createReadyNodegroup(t, f, "c1", "ng1", 2)
-
-	rec, p := ownedProfile(t, f, "c1", "ng1")
-	assert.NotEqual(t, fixtureNodeRoleName, rec.InstanceProfileName, "the profile is the group's own, not the role-named one")
-	assert.Equal(t, rec.InstanceProfileName, seenAtLaunch, "the name must be durable before any worker launches")
-	assert.Equal(t, "c1", profileTag(p, profileClusterTag))
-	assert.Equal(t, "ng1", profileTag(p, profileNodegroupTag))
-	assert.True(t, profileHasTagValue(p, rec.InstanceProfileID), "a tag must carry the incarnation id")
-	for _, tg := range p.Tags {
-		assert.True(t, strings.HasPrefix(aws.StringValue(tg.Key), "spinifex:"), "tag %q: aws: keys are reserved, ownership tags must be spinifex:", aws.StringValue(tg.Key))
-	}
-	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(p))
-
-	require.Equal(t, 2, f.worker.runCount())
-	for _, in := range f.worker.runCalls {
-		require.NotNil(t, in.IamInstanceProfile)
-		assert.True(t, strings.HasSuffix(aws.StringValue(in.IamInstanceProfile.Arn), "/"+rec.InstanceProfileName),
-			"every worker launches with the group's own profile, got %s", aws.StringValue(in.IamInstanceProfile.Arn))
-	}
-}
-
-func TestCreateNodegroup_TwoGroupsOnOneRoleGetTwoProfiles(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createUnreadyNodegroup(t, f, "c1", "ng-a", 1)
-	createUnreadyNodegroup(t, f, "c1", "ng-b", 1)
-
-	recA, pA := ownedProfile(t, f, "c1", "ng-a")
-	recB, pB := ownedProfile(t, f, "c1", "ng-b")
-	assert.NotEqual(t, recA.InstanceProfileName, recB.InstanceProfileName)
-	assert.NotEqual(t, recA.InstanceProfileID, recB.InstanceProfileID)
-	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(pA))
-	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(pB))
-}
-
-func TestCreateNodegroup_RecreatedGroupOfSameNameGetsANewProfile(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createUnreadyNodegroup(t, f, "c1", "ng1", 1)
-	first, _ := ownedProfile(t, f, "c1", "ng1")
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-	sweepNodegroups(t, f)
-
-	createUnreadyNodegroup(t, f, "c1", "ng1", 1)
-	second, _ := ownedProfile(t, f, "c1", "ng1")
-	assert.NotEqual(t, first.InstanceProfileName, second.InstanceProfileName)
-	assert.NotEqual(t, first.InstanceProfileID, second.InstanceProfileID)
-	_, stillThere := fixtureIAM(t, f).Profiles[first.InstanceProfileName]
-	assert.False(t, stillThere, "the first incarnation's profile must have gone with it")
-}
-
-func TestDeleteNodegroup_AfterCreateFailedRemovesProfileAndGroup(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createUnreadyNodegroup(t, f, "c1", "ng1", 2)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-	require.Equal(t, eks.NodegroupStatusCreateFailed, rec.Status)
-
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-	sweepNodegroups(t, f)
-
-	_, ok := fixtureIAM(t, f).Profiles[rec.InstanceProfileName]
-	assert.False(t, ok)
-	assert.Equal(t, 2, f.worker.terminatedCount())
-	_, err = describeNodegroupCall(f, "c1", "ng1")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-}
-
-func TestDeleteNodegroup_ProfileSurvivesWhileWorkersAreLive(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-	f.worker.termErr = errors.New("terminate refused")
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	sweepNodegroups(t, f)
-
-	m := fixtureIAM(t, f)
-	assert.Contains(t, m.Profiles, rec.InstanceProfileName, "a profile in use by live workers must not be touched")
-	assert.Zero(t, m.RemoveRoleCalls)
-	assert.Zero(t, m.DeleteInstanceProfileCalls)
-}
-
-func TestDeleteNodegroup_PartialProfileDeleteConvergesOnRetry(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-	m := fixtureIAM(t, f)
-	m.RemoveRoleFromInstanceProfileErr = errors.New("iam unavailable")
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	sweepNodegroups(t, f)
-	still, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
-	require.NoError(t, err, "the group is not gone while its profile is")
-	assert.Equal(t, eks.NodegroupStatusDeleting, still.Status)
-
-	m.RemoveRoleFromInstanceProfileErr = nil
-	backdateNodegroupReap(t, f, "c1", "ng1")
-	sweepNodegroups(t, f)
-	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
-	_, err = describeNodegroupCall(f, "c1", "ng1")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-}
-
-func TestDeleteNodegroup_CustomerRoleNamedProfileSurvives(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	seedCustomerProfile(f, t, fixtureNodeRoleName, fixtureNodeRoleName, nil)
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-	sweepNodegroups(t, f)
-
-	m := fixtureIAM(t, f)
-	assert.NotContains(t, m.Profiles, rec.InstanceProfileName, "the group's own profile goes")
-	require.Contains(t, m.Profiles, fixtureNodeRoleName, "a customer's profile is never EKS's to delete")
-	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[fixtureNodeRoleName]))
-	assert.Equal(t, 1, m.DeleteInstanceProfileCalls)
-}
-
-func TestDeleteNodegroup_RecordedProfileMissingOwnershipTagSurvives(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-	m := fixtureIAM(t, f)
-	_, err := m.UntagInstanceProfile(testAccountID, &iam.UntagInstanceProfileInput{
-		InstanceProfileName: aws.String(rec.InstanceProfileName),
-		TagKeys:             aws.StringSlice([]string{profileClusterTag}),
-	})
-	require.NoError(t, err)
-
-	_, err = deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-	sweepNodegroups(t, f)
-
-	require.Contains(t, m.Profiles, rec.InstanceProfileName)
-	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[rec.InstanceProfileName]))
-	assert.Zero(t, m.RemoveRoleCalls)
-	_, err = describeNodegroupCall(f, "c1", "ng1")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-}
-
-func TestDeleteNodegroup_OldLayoutGroupDeletesAndLeavesRoleNamedProfile(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	seedCustomerProfile(f, t, fixtureNodeRoleName, fixtureNodeRoleName, nil)
-	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{
-		ClusterName: "c1", Name: "old", Status: eks.NodegroupStatusActive,
-		InstanceIDs: []string{"i-old1"}, NodeRole: fixtureNodeRoleARN,
-	}))
-
-	_, err := deleteNodegroupCall(f, "c1", "old")
-	require.NoError(t, err)
-	sweepNodegroups(t, f)
-
-	require.Len(t, f.worker.terminateCalls, 1)
-	assert.Equal(t, []string{"i-old1"}, f.worker.terminateCalls[0])
-	_, err = describeNodegroupCall(f, "c1", "old")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-	m := fixtureIAM(t, f)
-	require.Contains(t, m.Profiles, fixtureNodeRoleName)
-	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[fixtureNodeRoleName]))
-	assert.Zero(t, m.RemoveRoleCalls)
-	assert.Zero(t, m.DeleteInstanceProfileCalls)
-}
-
-// No path deletes a profile unless it is recorded, tagged for this
-// incarnation, holds the record's node role and is named by no other record.
-func TestInvariant_NodegroupDeleteNeverRemovesAProfileItDoesNotOwn(t *testing.T) {
-	t.Parallel()
-	ownership := map[string]string{profileClusterTag: "c1", profileNodegroupTag: "ng1"}
-
-	t.Run("customer pre-creates a profile at a guessed name", func(t *testing.T) {
-		t.Parallel()
-		f := newEKSServiceFixture(t)
-		seedActiveClusterWithToken(t, f, "c1")
-		guesses := []string{
-			"eks-" + arn.EKSNodegroupDiscriminator(testAccountID, "c1", "ng1"),
-			"eks-c1-ng1",
-			"c1-ng1",
-			fixtureNodeRoleName,
-		}
-		for _, name := range guesses {
-			seedCustomerProfile(f, t, name, fixtureNodeRoleName, ownership)
-		}
-		createReadyNodegroup(t, f, "c1", "ng1", 1)
-		rec, _ := ownedProfile(t, f, "c1", "ng1")
-		require.NotContains(t, guesses, rec.InstanceProfileName, "the name must not be guessable")
-
-		_, err := deleteNodegroupCall(f, "c1", "ng1")
-		require.NoError(t, err)
-		sweepNodegroups(t, f)
-
-		m := fixtureIAM(t, f)
-		for _, name := range guesses {
-			require.Contains(t, m.Profiles, name, "customer profile %q was deleted", name)
-			assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[name]), "customer profile %q lost its role", name)
-		}
-	})
-
-	t.Run("recorded profile was swapped for one holding another role", func(t *testing.T) {
-		t.Parallel()
-		f := newEKSServiceFixture(t)
-		seedActiveClusterWithToken(t, f, "c1")
-		createReadyNodegroup(t, f, "c1", "ng1", 1)
-		rec, p := ownedProfile(t, f, "c1", "ng1")
-		p.Roles = []*iam.Role{{RoleName: aws.String("customer-role")}}
-
-		_, err := deleteNodegroupCall(f, "c1", "ng1")
-		require.NoError(t, err)
-		sweepNodegroups(t, f)
-
-		m := fixtureIAM(t, f)
-		require.Contains(t, m.Profiles, rec.InstanceProfileName)
-		assert.Equal(t, []string{"customer-role"}, profileRoleNames(m.Profiles[rec.InstanceProfileName]))
-		assert.Zero(t, m.RemoveRoleCalls)
-		assert.Zero(t, m.DeleteInstanceProfileCalls)
-	})
-
-	t.Run("another nodegroup record names the same profile", func(t *testing.T) {
-		t.Parallel()
-		f := newEKSServiceFixture(t)
-		seedActiveClusterWithToken(t, f, "c1")
-		createReadyNodegroup(t, f, "c1", "ng1", 1)
-		rec, _ := ownedProfile(t, f, "c1", "ng1")
-		require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{
-			ClusterName: "c1", Name: "ng2", Status: eks.NodegroupStatusActive, NodeRole: fixtureNodeRoleARN,
-			InstanceProfileName: rec.InstanceProfileName, InstanceProfileID: rec.InstanceProfileID,
-		}))
-
-		_, err := deleteNodegroupCall(f, "c1", "ng1")
-		require.NoError(t, err)
-		sweepNodegroups(t, f)
-
-		m := fixtureIAM(t, f)
-		require.Contains(t, m.Profiles, rec.InstanceProfileName)
-		assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[rec.InstanceProfileName]))
-		assert.Zero(t, m.DeleteInstanceProfileCalls)
-	})
-}
-
-func TestDeleteNodegroup_ProfileAndRecordKeptUntilWorkersProvenGone(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 2)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-	f.worker.stillLive = []string{rec.InstanceIDs[0]}
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	sweepNodegroups(t, f)
-
-	m := fixtureIAM(t, f)
-	assert.Contains(t, m.Profiles, rec.InstanceProfileName)
-	assert.Zero(t, m.RemoveRoleCalls)
-	assert.Zero(t, m.DeleteInstanceProfileCalls)
-	got, err := describeNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-	assert.Equal(t, eks.NodegroupStatusDeleting, aws.StringValue(got.Nodegroup.Status))
-	cur, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
-	require.NoError(t, err)
-	assert.Equal(t, rec.InstanceIDs, cur.InstanceIDs, "worker ids stay recorded until they are proven gone")
-	assert.Zero(t, cur.DeleteReapAttempts, "workers still ending is not a failed pass")
-
-	f.worker.mu.Lock()
-	f.worker.stillLive = nil
-	f.worker.mu.Unlock()
-	sweepNodegroups(t, f)
-	_, err = describeNodegroupCall(f, "c1", "ng1")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
-}
-
-func TestDeleteNodegroup_UndecidableWorkerStateKeepsProfile(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-	f.worker.liveErr = errors.New("describe fan-out did not complete")
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	sweepNodegroups(t, f)
-
-	m := fixtureIAM(t, f)
-	assert.Contains(t, m.Profiles, rec.InstanceProfileName)
-	assert.Zero(t, m.DeleteInstanceProfileCalls)
-}
-
-func TestDeleteNodegroup_KickFinishesTheDeleteWithoutAReaperSweep(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 1)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-	f.svc.noDeleteKick = false
-
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-	f.svc.WaitNodegroupDeletes()
-
-	_, err = describeNodegroupCall(f, "c1", "ng1")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-	assert.NotContains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
-}
-
-func TestDeleteNodegroup_WorkerStuckPastDeadlineEndsDeleteFailedKeepingProfile(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	createReadyNodegroup(t, f, "c1", "ng1", 2)
-	rec, _ := ownedProfile(t, f, "c1", "ng1")
-	stuck := rec.InstanceIDs[0]
-	f.worker.stillLive = []string{stuck}
-	_, err := deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-
-	sweepNodegroups(t, f)
-	cur, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
-	require.NoError(t, err)
-	require.Equal(t, eks.NodegroupStatusDeleting, cur.Status, "within the deadline the group keeps waiting")
-
-	backdateNodegroupReap(t, f, "c1", "ng1")
-	sweepNodegroups(t, f)
-
-	desc, err := describeNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-	assert.Equal(t, eks.NodegroupStatusDeleteFailed, aws.StringValue(desc.Nodegroup.Status))
-	require.NotEmpty(t, desc.Nodegroup.Health.Issues)
-	issue := desc.Nodegroup.Health.Issues[0]
-	assert.Equal(t, eks.NodegroupIssueCodeNodeTerminationFailure, aws.StringValue(issue.Code))
-	assert.Contains(t, aws.StringValue(issue.Message), stuck)
-	assert.Contains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
-
-	f.worker.mu.Lock()
-	f.worker.stillLive = nil
-	f.worker.mu.Unlock()
-	_, err = deleteNodegroupCall(f, "c1", "ng1")
-	require.NoError(t, err)
-	sweepNodegroups(t, f)
-	_, err = describeNodegroupCall(f, "c1", "ng1")
-	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
-	assert.NotContains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
-}

spinifex/domains/eks/cluster/nodegroup_profile.go

@@ -1,174 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"fmt"
-	"log/slog"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/iam"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-const (
-	nodegroupProfilePrefix = "eks-"
-
-	profileClusterTagKey   = "spinifex:eks-cluster"
-	profileNodegroupTagKey = "spinifex:eks-nodegroup"
-	profileIDTagKey        = "spinifex:eks-profile-id"
-)
-
-// nodeProfileForLaunch returns the ARN of the instance profile a worker of rec
-// launches with. A record that names no profile of its own predates per-group
-// profiles and keeps the role-named one it already uses.
-func (s *EKSServiceImpl) nodeProfileForLaunch(accountID string, rec *NodegroupRecord) (string, error) {
-	if rec.InstanceProfileName == "" {
-		return s.ensureNodeInstanceProfile(accountID, rec.NodeRole)
-	}
-	return s.ensureNodegroupProfile(accountID, rec)
-}
-
-func nodegroupProfileTags(rec *NodegroupRecord) []*iam.Tag {
-	return []*iam.Tag{
-		{Key: aws.String(profileClusterTagKey), Value: aws.String(rec.ClusterName)},
-		{Key: aws.String(profileNodegroupTagKey), Value: aws.String(rec.Name)},
-		{Key: aws.String(profileIDTagKey), Value: aws.String(rec.InstanceProfileID)},
-	}
-}
-
-// profileOwnedBy reports whether p carries the ownership tags of this exact
-// incarnation of rec.
-func profileOwnedBy(p *iam.InstanceProfile, rec *NodegroupRecord) bool {
-	have := map[string]string{}
-	for _, t := range p.Tags {
-		have[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
-	}
-	for _, want := range nodegroupProfileTags(rec) {
-		if have[aws.StringValue(want.Key)] != aws.StringValue(want.Value) {
-			return false
-		}
-	}
-	return true
-}
-
-func roleNameFromARN(roleARN string) string {
-	return roleARN[strings.LastIndex(roleARN, "/")+1:]
-}
-
-// ensureNodegroupProfile finds or creates the instance profile recorded for
-// rec, tagged as its own, and makes sure it holds the node role. Concurrent
-// worker launches converge on the one profile.
-func (s *EKSServiceImpl) ensureNodegroupProfile(accountID string, rec *NodegroupRecord) (string, error) {
-	roleName, err := s.resolveNodeRoleName(accountID, rec.NodeRole)
-	if err != nil {
-		return "", err
-	}
-	name := rec.InstanceProfileName
-	ens := s.iamEnsurer()
-
-	got, err := ens.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String(name)})
-	if err != nil {
-		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
-			return "", fmt.Errorf("get instance profile %q: %w", name, err)
-		}
-		_, cerr := ens.CreateInstanceProfile(accountID, &iam.CreateInstanceProfileInput{
-			InstanceProfileName: aws.String(name),
-			Tags:                nodegroupProfileTags(rec),
-		})
-		if cerr != nil {
-			// A racing launch may have created it first; re-read and converge.
-			if code, _ := awserrors.ResolveErrorCode(cerr); code != awserrors.ErrorIAMEntityAlreadyExists {
-				return "", fmt.Errorf("create instance profile %q: %w", name, cerr)
-			}
-		}
-		got, err = ens.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String(name)})
-		if err != nil {
-			return "", fmt.Errorf("re-get instance profile %q: %w", name, err)
-		}
-	}
-	if !profileOwnedBy(got.InstanceProfile, rec) {
-		return "", fmt.Errorf("instance profile %q exists but is not this nodegroup's", name)
-	}
-	return s.attachRoleToProfile(accountID, name, roleName,
-		aws.StringValue(got.InstanceProfile.Arn), len(got.InstanceProfile.Roles) > 0)
-}
-
-// deleteNodegroupProfile removes the profile recorded for rec, and only that
-// one: it must be recorded, tagged for this incarnation, hold exactly the node
-// role and be named by no other record. Anything else is left alone.
-func (s *EKSServiceImpl) deleteNodegroupProfile(ctx context.Context, accountID string, kv jetstream.KeyValue, rec *NodegroupRecord) error {
-	name := rec.InstanceProfileName
-	ens := s.iamEnsurer()
-	if name == "" || ens == nil {
-		return nil
-	}
-	got, err := ens.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String(name)})
-	if err != nil {
-		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
-			return nil
-		}
-		return fmt.Errorf("get instance profile %q: %w", name, err)
-	}
-	p := got.InstanceProfile
-
-	if !profileOwnedBy(p, rec) {
-		slog.WarnContext(ctx, "eks: leaving instance profile whose ownership tags do not match its nodegroup",
-			"cluster", rec.ClusterName, "nodegroup", rec.Name, "profile", name)
-		return nil
-	}
-	nodeRole := roleNameFromARN(rec.NodeRole)
-	if len(p.Roles) > 1 || (len(p.Roles) == 1 && aws.StringValue(p.Roles[0].RoleName) != nodeRole) {
-		slog.WarnContext(ctx, "eks: leaving instance profile that holds a role other than the node role",
-			"cluster", rec.ClusterName, "nodegroup", rec.Name, "profile", name)
-		return nil
-	}
-	shared, err := s.profileNamedByOtherRecord(ctx, kv, rec)
-	if err != nil {
-		return err
-	}
-	if shared {
-		slog.WarnContext(ctx, "eks: leaving instance profile another nodegroup record names",
-			"cluster", rec.ClusterName, "nodegroup", rec.Name, "profile", name)
-		return nil
-	}
-
-	if len(p.Roles) == 1 {
-		_, err := ens.RemoveRoleFromInstanceProfile(accountID, &iam.RemoveRoleFromInstanceProfileInput{
-			InstanceProfileName: aws.String(name), RoleName: aws.String(nodeRole),
-		})
-		if err != nil {
-			if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
-				return fmt.Errorf("remove role %q from instance profile %q: %w", nodeRole, name, err)
-			}
-		}
-	}
-	if _, err := ens.DeleteInstanceProfile(accountID, &iam.DeleteInstanceProfileInput{InstanceProfileName: aws.String(name)}); err != nil {
-		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
-			return fmt.Errorf("delete instance profile %q: %w", name, err)
-		}
-	}
-	return nil
-}
-
-// profileNamedByOtherRecord reports whether any nodegroup record in the
-// account other than rec names rec's profile.
-func (s *EKSServiceImpl) profileNamedByOtherRecord(ctx context.Context, kv jetstream.KeyValue, rec *NodegroupRecord) (bool, error) {
-	clusters, err := listClusterNames(ctx, kv)
-	if err != nil {
-		return false, fmt.Errorf("list clusters: %w", err)
-	}
-	for _, cluster := range clusters {
-		recs, err := ListNodegroupRecords(ctx, kv, cluster)
-		if err != nil {
-			return false, fmt.Errorf("list nodegroups of %s: %w", cluster, err)
-		}
-		for _, other := range recs {
-			if other.InstanceProfileName == rec.InstanceProfileName && (other.ClusterName != rec.ClusterName || other.Name != rec.Name) {
-				return true, nil
-			}
-		}
-	}
-	return false, nil
-}

spinifex/domains/eks/cluster/nodegroup_test.go

@@ -86,12 +86,6 @@ type fakeWorkerLauncher struct {
 	// before subsequent calls succeed — simulates a transient launch failure
 	// (QMP timeout, host pressure) that clears on retry.
 	failNextN int
-	// beforeRun, when set, runs at the start of every launch, before it is
-	// recorded, so a test can inspect durable state at launch time.
-	beforeRun func(*ec2.RunInstancesInput)
-	// stillLive lists instances LiveWorkerInstances keeps reporting as live.
-	stillLive []string
-	liveErr   error
 }
 
 var _ WorkerLauncher = (*fakeWorkerLauncher)(nil)
@@ -107,9 +101,6 @@ func (f *fakeWorkerLauncher) RunWorkerInstance(ctx context.Context, input *ec2.R
 func (f *fakeWorkerLauncher) RunWorkerInstanceOnNode(_ context.Context, nodeID string, input *ec2.RunInstancesInput, _ string) (*ec2.Reservation, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	if f.beforeRun != nil {
-		f.beforeRun(input)
-	}
 	f.runCalls = append(f.runCalls, input)
 	f.runNodes = append(f.runNodes, nodeID)
 	if f.failNextN > 0 {
@@ -138,21 +129,6 @@ func (f *fakeWorkerLauncher) TerminateWorkerInstances(_ context.Context, ids []s
 	return f.termErr
 }
 
-func (f *fakeWorkerLauncher) LiveWorkerInstances(_ context.Context, ids []string, _ string) ([]string, error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if f.liveErr != nil {
-		return nil, f.liveErr
-	}
-	var live []string
-	for _, id := range ids {
-		if slices.Contains(f.stillLive, id) {
-			live = append(live, id)
-		}
-	}
-	return live, nil
-}
-
 // runCount / terminatedCount return the recorded call totals under the lock so
 // concurrent tests can read them safely after joining.
 func (f *fakeWorkerLauncher) runCount() int {
@@ -951,7 +927,7 @@ func TestUpdateNodegroupConfig_CapacityErrorSurfacesCode(t *testing.T) {
 	})
 }
 
-func TestDeleteNodegroup_ReapedGroupIsGoneAndRepeatDeleteIsNotFound(t *testing.T) {
+func TestDeleteNodegroup_TerminatesAndIsIdempotent(t *testing.T) {
 	t.Parallel()
 	f := newEKSServiceFixture(t)
 	seedActiveClusterWithToken(t, f, "c1")
@@ -964,16 +940,13 @@ func TestDeleteNodegroup_ReapedGroupIsGoneAndRepeatDeleteIsNotFound(t *testing.T
 		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1"),
 	}, testAccountID)
 	require.NoError(t, err)
-	_, err = f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
-	require.NoError(t, err)
-	f.svc.WaitNodegroupDeletes()
 	require.Len(t, f.worker.terminateCalls, 1)
 	assert.Len(t, f.worker.terminateCalls[0], 2)
 
 	_, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
 	assert.ErrorIs(t, err, ErrNodegroupNotFound)
 
-	// Once the reaper has finished the group is gone: ResourceNotFoundException.
+	// Second delete → ResourceNotFoundException (record already gone).
 	_, err = f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
 		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1"),
 	}, testAccountID)

spinifex/domains/eks/cluster/service_impl.go

@@ -201,26 +201,20 @@ type WorkerLauncher interface {
 	// host spread. An empty nodeID launches on the local node like RunWorkerInstance.
 	RunWorkerInstanceOnNode(ctx context.Context, nodeID string, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error)
 	TerminateWorkerInstances(ctx context.Context, instanceIDs []string, accountID string) error
-	// LiveWorkerInstances returns the given workers not yet proven terminated or
-	// absent. An undecidable answer is an error, never an empty list.
-	LiveWorkerInstances(ctx context.Context, instanceIDs []string, accountID string) ([]string, error)
 }
 
 // iamEnsurer resolves the IAM ensurer, preferring the test-injected deps.IAM,
 // then the lazy IAMProvider (built at launch time to dodge the daemon-startup
 // NATS-KV race). Returns nil when neither yields a service, so callers fall back
-// to static creds / skip the profile.
+// to static creds / skip the profile. The provider's concrete service satisfies
+// instanceProfileEnsurer (identical method set to SystemInstanceRoleEnsurer).
 func (s *EKSServiceImpl) iamEnsurer() instanceProfileEnsurer {
 	if s.deps.IAM != nil {
 		return s.deps.IAM
 	}
 	if s.deps.IAMProvider != nil {
 		if e := s.deps.IAMProvider(); e != nil {
-			// The concrete service has the wider surface; the shared interface
-			// stays narrow so other callers are not forced to grow it.
-			if ep, ok := e.(instanceProfileEnsurer); ok {
-				return ep
-			}
+			return e
 		}
 	}
 	return nil
@@ -233,8 +227,6 @@ type instanceProfileEnsurer interface {
 	GetInstanceProfile(accountID string, input *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error)
 	CreateInstanceProfile(accountID string, input *iam.CreateInstanceProfileInput) (*iam.CreateInstanceProfileOutput, error)
 	AddRoleToInstanceProfile(accountID string, input *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error)
-	RemoveRoleFromInstanceProfile(accountID string, input *iam.RemoveRoleFromInstanceProfileInput) (*iam.RemoveRoleFromInstanceProfileOutput, error)
-	DeleteInstanceProfile(accountID string, input *iam.DeleteInstanceProfileInput) (*iam.DeleteInstanceProfileOutput, error)
 
 	// Role find-or-create surface for the system-managed control-plane role.
 	// Unlike worker node roles (customer-supplied), the k3s server role is
@@ -284,12 +276,6 @@ type EKSServiceImpl struct {
 	teardownMu      sync.Mutex
 	teardownRunning map[string]struct{}
 	teardownWG      sync.WaitGroup
-	// noDeleteKick, set by tests, leaves a nodegroup delete to the reaper sweep.
-	noDeleteKick bool
-	// workersEndWait and workersEndPoll bound how long one delete pass waits for
-	// ended workers to be proven gone; zero means the defaults.
-	workersEndWait time.Duration
-	workersEndPoll time.Duration
 	// teardownKV, when set by a test, replaces the converger's account bucket.
 	teardownKV func(ctx context.Context, accountID string) (jetstream.KeyValue, error)
 

spinifex/domains/eks/cluster/types.go

@@ -49,16 +49,6 @@ type NodegroupRecord struct {
 	Tags           map[string]string `json:"tags,omitempty"`
 	InstanceIDs    []string          `json:"instanceIds,omitempty"`
 	Health         string            `json:"health,omitempty"`
-	// InstanceProfileID and InstanceProfileName identify the instance profile
-	// EKS made for this incarnation of the nodegroup; both are empty on a record
-	// that predates it.
-	InstanceProfileID   string `json:"instanceProfileId,omitempty"`
-	InstanceProfileName string `json:"instanceProfileName,omitempty"`
-	// DeletingSince, DeleteReapAttempts and LastDeleteReapAttempt pace the reaper that finishes
-	// a DELETING nodegroup.
-	DeletingSince         time.Time `json:"deletingSince,omitzero"`
-	DeleteReapAttempts    int       `json:"deleteReapAttempts,omitempty"`
-	LastDeleteReapAttempt time.Time `json:"lastDeleteReapAttempt,omitzero"`
 	// GPUEnabled/GPUVendor cache whether InstanceTypes includes a GPU family and
 	// which vendor, so worker launches resolve the matching GPU node AMI without
 	// re-scanning InstanceTypes on every launch.

spinifex/domains/eks/cluster/worker_membership_test.go

@@ -56,9 +56,6 @@ func TestListManagedWorkers_DropsWorkersTheNodegroupTerminated(t *testing.T) {
 		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng2"),
 	}, testAccountID)
 	require.NoError(t, err)
-	_, err = f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
-	require.NoError(t, err)
-	f.svc.WaitNodegroupDeletes()
 	got = managedWorkers(t, f, "c1")
 	assert.Len(t, got, 1)
 	for _, id := range ng2.InstanceIDs {

spinifex/domains/iam/mock/mock.go

@@ -36,9 +36,6 @@ type SystemInstanceRoleEnsurer struct {
 	CreateRoleCalls               int
 	CreateInstanceProfileCalls    int
 	AddRoleToInstanceProfileCalls int
-	TagInstanceProfileCalls       int
-	RemoveRoleCalls               int
-	DeleteInstanceProfileCalls    int
 
 	// LastTrustDoc captures the AssumeRolePolicyDocument of the most recent
 	// CreateRole call.
@@ -57,14 +54,6 @@ type SystemInstanceRoleEnsurer struct {
 	// AddRoleToInstanceProfileErr, when set, is returned by
 	// AddRoleToInstanceProfile instead of attaching.
 	AddRoleToInstanceProfileErr error
-	// RemoveRoleFromInstanceProfileErr and DeleteInstanceProfileErr, when set,
-	// are returned by the matching call instead of acting.
-	RemoveRoleFromInstanceProfileErr error
-	DeleteInstanceProfileErr         error
-	// GetInstanceProfileErr, when set, is returned by GetInstanceProfile
-	// instead of reading, for exercising callers that must tell a failed read
-	// from a missing profile.
-	GetInstanceProfileErr error
 	// EmptyInstanceProfileARN makes CreateInstanceProfile return an empty
 	// ARN, for exercising callers that must reject it.
 	EmptyInstanceProfileARN bool
@@ -92,7 +81,6 @@ func copyRole(r *iam.Role) *iam.Role {
 func copyProfile(p *iam.InstanceProfile) *iam.InstanceProfile {
 	c := *p
 	c.Roles = append([]*iam.Role(nil), p.Roles...)
-	c.Tags = append([]*iam.Tag(nil), p.Tags...)
 	return &c
 }
 
@@ -142,9 +130,6 @@ func (e *SystemInstanceRoleEnsurer) GetInstanceProfile(_ string, in *iam.GetInst
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
-	if e.GetInstanceProfileErr != nil {
-		return nil, e.GetInstanceProfileErr
-	}
 	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
 	if !ok {
 		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
@@ -172,7 +157,6 @@ func (e *SystemInstanceRoleEnsurer) CreateInstanceProfile(accountID string, in *
 	p := &iam.InstanceProfile{
 		InstanceProfileName: aws.String(name),
 		Arn:                 aws.String(arn),
-		Tags:                append([]*iam.Tag(nil), in.Tags...),
 	}
 	e.Profiles[name] = p
 	return &iam.CreateInstanceProfileOutput{InstanceProfile: copyProfile(p)}, nil
@@ -197,101 +181,3 @@ func (e *SystemInstanceRoleEnsurer) AddRoleToInstanceProfile(_ string, in *iam.A
 	p.Roles = []*iam.Role{{RoleName: in.RoleName}}
 	return &iam.AddRoleToInstanceProfileOutput{}, nil
 }
-
-func (e *SystemInstanceRoleEnsurer) TagInstanceProfile(_ string, in *iam.TagInstanceProfileInput) (*iam.TagInstanceProfileOutput, error) {
-	e.mu.Lock()
-	defer e.mu.Unlock()
-
-	e.TagInstanceProfileCalls++
-	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
-	if !ok {
-		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
-	}
-	for _, t := range in.Tags {
-		replaced := false
-		for i, have := range p.Tags {
-			if aws.StringValue(have.Key) == aws.StringValue(t.Key) {
-				p.Tags[i] = t
-				replaced = true
-				break
-			}
-		}
-		if !replaced {
-			p.Tags = append(p.Tags, t)
-		}
-	}
-	return &iam.TagInstanceProfileOutput{}, nil
-}
-
-func (e *SystemInstanceRoleEnsurer) UntagInstanceProfile(_ string, in *iam.UntagInstanceProfileInput) (*iam.UntagInstanceProfileOutput, error) {
-	e.mu.Lock()
-	defer e.mu.Unlock()
-
-	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
-	if !ok {
-		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
-	}
-	drop := map[string]bool{}
-	for _, k := range in.TagKeys {
-		drop[aws.StringValue(k)] = true
-	}
-	kept := p.Tags[:0:0]
-	for _, t := range p.Tags {
-		if !drop[aws.StringValue(t.Key)] {
-			kept = append(kept, t)
-		}
-	}
-	p.Tags = kept
-	return &iam.UntagInstanceProfileOutput{}, nil
-}
-
-func (e *SystemInstanceRoleEnsurer) ListInstanceProfileTags(_ string, in *iam.ListInstanceProfileTagsInput) (*iam.ListInstanceProfileTagsOutput, error) {
-	e.mu.Lock()
-	defer e.mu.Unlock()
-
-	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
-	if !ok {
-		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
-	}
-	return &iam.ListInstanceProfileTagsOutput{Tags: append([]*iam.Tag(nil), p.Tags...)}, nil
-}
-
-func (e *SystemInstanceRoleEnsurer) RemoveRoleFromInstanceProfile(_ string, in *iam.RemoveRoleFromInstanceProfileInput) (*iam.RemoveRoleFromInstanceProfileOutput, error) {
-	e.mu.Lock()
-	defer e.mu.Unlock()
-
-	e.RemoveRoleCalls++
-	if e.RemoveRoleFromInstanceProfileErr != nil {
-		return nil, e.RemoveRoleFromInstanceProfileErr
-	}
-	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
-	if !ok {
-		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
-	}
-	if len(p.Roles) == 0 || aws.StringValue(p.Roles[0].RoleName) != aws.StringValue(in.RoleName) {
-		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The role with name %s cannot be found.", aws.StringValue(in.RoleName))
-	}
-	p.Roles = nil
-	return &iam.RemoveRoleFromInstanceProfileOutput{}, nil
-}
-
-// DeleteInstanceProfile refuses while a role is attached, as IAM does.
-func (e *SystemInstanceRoleEnsurer) DeleteInstanceProfile(_ string, in *iam.DeleteInstanceProfileInput) (*iam.DeleteInstanceProfileOutput, error) {
-	e.mu.Lock()
-	defer e.mu.Unlock()
-
-	e.DeleteInstanceProfileCalls++
-	if e.DeleteInstanceProfileErr != nil {
-		return nil, e.DeleteInstanceProfileErr
-	}
-	name := aws.StringValue(in.InstanceProfileName)
-	p, ok := e.Profiles[name]
-	if !ok {
-		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", name)
-	}
-	if len(p.Roles) > 0 {
-		return nil, awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first.")
-	}
-	delete(e.Profiles, name)
-	return &iam.DeleteInstanceProfileOutput{}, nil
-}

spinifex/domains/iam/mock/mock_test.go

@@ -83,112 +83,3 @@ func TestPutRolePolicy_InjectedErrorStillRecordsCall(t *testing.T) {
 		t.Fatalf("RolePolicies[r1] = %q, want %q", e.RolePolicies["r1"], "doc")
 	}
 }
-
-func seedProfile(t *testing.T, e *mock.SystemInstanceRoleEnsurer, name string) {
-	t.Helper()
-	if _, err := e.CreateInstanceProfile("acct", &iam.CreateInstanceProfileInput{InstanceProfileName: aws.String(name)}); err != nil {
-		t.Fatalf("CreateInstanceProfile: %v", err)
-	}
-}
-
-func requireCode(t *testing.T, err error, want string) {
-	t.Helper()
-	if err == nil || awserrors.ValidErrorCodeFromError(err) != want {
-		t.Fatalf("err = %v, want %s", err, want)
-	}
-}
-
-func TestInstanceProfileTags_TagReplacesSameKeyUntagDropsListReturnsCopy(t *testing.T) {
-	e := mock.New()
-	seedProfile(t, e, "p1")
-	tag := func(k, v string) *iam.Tag { return &iam.Tag{Key: aws.String(k), Value: aws.String(v)} }
-	if _, err := e.TagInstanceProfile("acct", &iam.TagInstanceProfileInput{
-		InstanceProfileName: aws.String("p1"), Tags: []*iam.Tag{tag("a", "1"), tag("b", "2")},
-	}); err != nil {
-		t.Fatalf("TagInstanceProfile: %v", err)
-	}
-	if _, err := e.TagInstanceProfile("acct", &iam.TagInstanceProfileInput{
-		InstanceProfileName: aws.String("p1"), Tags: []*iam.Tag{tag("a", "9")},
-	}); err != nil {
-		t.Fatalf("re-tag: %v", err)
-	}
-	if _, err := e.UntagInstanceProfile("acct", &iam.UntagInstanceProfileInput{
-		InstanceProfileName: aws.String("p1"), TagKeys: aws.StringSlice([]string{"b"}),
-	}); err != nil {
-		t.Fatalf("UntagInstanceProfile: %v", err)
-	}
-	out, err := e.ListInstanceProfileTags("acct", &iam.ListInstanceProfileTagsInput{InstanceProfileName: aws.String("p1")})
-	if err != nil {
-		t.Fatalf("ListInstanceProfileTags: %v", err)
-	}
-	if len(out.Tags) != 1 || aws.StringValue(out.Tags[0].Key) != "a" || aws.StringValue(out.Tags[0].Value) != "9" {
-		t.Fatalf("tags = %v, want only a=9", out.Tags)
-	}
-}
-
-func TestInstanceProfileTags_MissingProfileIsNoSuchEntity(t *testing.T) {
-	e := mock.New()
-	name := aws.String("absent")
-	_, err := e.TagInstanceProfile("acct", &iam.TagInstanceProfileInput{InstanceProfileName: name})
-	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
-	_, err = e.UntagInstanceProfile("acct", &iam.UntagInstanceProfileInput{InstanceProfileName: name})
-	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
-	_, err = e.ListInstanceProfileTags("acct", &iam.ListInstanceProfileTagsInput{InstanceProfileName: name})
-	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
-}
-
-func TestDeleteInstanceProfile_RefusesWhileRoleAttachedThenDeletes(t *testing.T) {
-	e := mock.New()
-	seedProfile(t, e, "p1")
-	if _, err := e.AddRoleToInstanceProfile("acct", &iam.AddRoleToInstanceProfileInput{
-		InstanceProfileName: aws.String("p1"), RoleName: aws.String("r1"),
-	}); err != nil {
-		t.Fatalf("AddRoleToInstanceProfile: %v", err)
-	}
-	del := &iam.DeleteInstanceProfileInput{InstanceProfileName: aws.String("p1")}
-	_, err := e.DeleteInstanceProfile("acct", del)
-	requireCode(t, err, awserrors.ErrorIAMDeleteConflict)
-	if _, ok := e.Profiles["p1"]; !ok {
-		t.Fatal("a refused delete must leave the profile in place")
-	}
-
-	_, err = e.RemoveRoleFromInstanceProfile("acct", &iam.RemoveRoleFromInstanceProfileInput{
-		InstanceProfileName: aws.String("p1"), RoleName: aws.String("other"),
-	})
-	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
-	if _, err := e.RemoveRoleFromInstanceProfile("acct", &iam.RemoveRoleFromInstanceProfileInput{
-		InstanceProfileName: aws.String("p1"), RoleName: aws.String("r1"),
-	}); err != nil {
-		t.Fatalf("RemoveRoleFromInstanceProfile: %v", err)
-	}
-	if _, err := e.DeleteInstanceProfile("acct", del); err != nil {
-		t.Fatalf("DeleteInstanceProfile: %v", err)
-	}
-	if _, ok := e.Profiles["p1"]; ok {
-		t.Fatal("profile still present after delete")
-	}
-	_, err = e.DeleteInstanceProfile("acct", del)
-	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
-}
-
-func TestInstanceProfileMutations_MissingProfileAndInjectedErrors(t *testing.T) {
-	e := mock.New()
-	name := aws.String("absent")
-	_, err := e.RemoveRoleFromInstanceProfile("acct", &iam.RemoveRoleFromInstanceProfileInput{InstanceProfileName: name, RoleName: aws.String("r")})
-	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
-
-	boom := errors.New("boom")
-	e.RemoveRoleFromInstanceProfileErr, e.DeleteInstanceProfileErr, e.GetInstanceProfileErr = boom, boom, boom
-	if _, err := e.RemoveRoleFromInstanceProfile("acct", &iam.RemoveRoleFromInstanceProfileInput{InstanceProfileName: name, RoleName: aws.String("r")}); !errors.Is(err, boom) {
-		t.Fatalf("RemoveRole err = %v, want injected", err)
-	}
-	if _, err := e.DeleteInstanceProfile("acct", &iam.DeleteInstanceProfileInput{InstanceProfileName: name}); !errors.Is(err, boom) {
-		t.Fatalf("Delete err = %v, want injected", err)
-	}
-	if _, err := e.GetInstanceProfile("acct", &iam.GetInstanceProfileInput{InstanceProfileName: name}); !errors.Is(err, boom) {
-		t.Fatalf("Get err = %v, want injected", err)
-	}
-	if e.RemoveRoleCalls != 2 || e.DeleteInstanceProfileCalls != 1 {
-		t.Fatalf("calls = remove %d delete %d, want 2 and 1", e.RemoveRoleCalls, e.DeleteInstanceProfileCalls)
-	}
-}

spinifex/domains/lambda/awsapi/get.go

@@ -23,42 +23,81 @@ func notFound(region, accountID, name string) error {
 		arn.FormatLambdaFunction(region, accountID, name))
 }
 
-// getFunction implements GetFunction for the $LATEST version.
-func (d Deps) getFunction(ctx context.Context, c *call) (reply, error) {
+// lookup resolves the requested $LATEST function's record; query is the
+// Qualifier query parameter, empty for an action that has none.
+func (d Deps) lookup(ctx context.Context, c *call, query string) (function.Record, uint64, error) {
 	name, q, err := resolveFunctionName(c.Region, c.AccountID, c.Params[0])
 	if err != nil {
-		return reply{}, err
+		return function.Record{}, 0, err
 	}
-	if query := c.Query.Get("Qualifier"); query != "" {
+	if query != "" {
 		if q != "" && q != query {
-			return reply{}, invalid("The derived qualifier from the function name does not match the specified qualifier.")
+			return function.Record{}, 0, invalid("The derived qualifier from the function name does not match the specified qualifier.")
 		}
 		q = query
 	}
 	if q != "" && q != "$LATEST" {
-		return reply{}, notFound(c.Region, c.AccountID, name+":"+q)
+		return function.Record{}, 0, notFound(c.Region, c.AccountID, name+":"+q)
 	}
 	rec, revision, err := d.Meta.Get(ctx, c.AccountID, c.Region, name)
 	if errors.Is(err, function.ErrNotFound) {
-		return reply{}, notFound(c.Region, c.AccountID, name)
+		return function.Record{}, 0, notFound(c.Region, c.AccountID, name)
 	}
+	return rec, revision, err
+}
+
+// loadConfiguration resolves the requested $LATEST function and renders its
+// configuration; it is a metadata read and never touches the code store.
+func (d Deps) loadConfiguration(ctx context.Context, c *call) (function.Record, *lambda.FunctionConfiguration, error) {
+	rec, revision, err := d.lookup(ctx, c, c.Query.Get("Qualifier"))
 	if err != nil {
-		return reply{}, err
+		return function.Record{}, nil, err
 	}
 
 	var env map[string]string
 	if rec.EnvCipher != "" {
 		if d.AEAD == nil {
-			return reply{}, serviceError("environment decryption is unavailable")
+			return function.Record{}, nil, serviceError("environment decryption is unavailable")
 		}
 		if env, err = function.OpenEnv(d.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher); err != nil {
 			slog.ErrorContext(ctx, "Lambda: open environment failed", "function_id", rec.FunctionID, "err", err)
-			return reply{}, serviceError("could not read the function environment")
+			return function.Record{}, nil, serviceError("could not read the function environment")
 		}
 	}
 	cfg, err := configuration(rec, revision, env)
 	if err != nil {
-		return reply{}, serviceError("could not build the function configuration")
+		return function.Record{}, nil, serviceError("could not build the function configuration")
+	}
+	return rec, cfg, nil
+}
+
+// getFunctionConfiguration implements GetFunctionConfiguration for $LATEST.
+func (d Deps) getFunctionConfiguration(ctx context.Context, c *call) (reply, error) {
+	_, cfg, err := d.loadConfiguration(ctx, c)
+	if err != nil {
+		return reply{}, err
+	}
+	return reply{Status: http.StatusOK, Body: cfg}, nil
+}
+
+// noCodeSigning is AWS's body for a function without a code signing config:
+// both members present and null, which the SDK shapes cannot encode.
+var noCodeSigning = rawJSON(`{"CodeSigningConfigArn":null,"FunctionName":null}`)
+
+// getFunctionCodeSigningConfig implements GetFunctionCodeSigningConfig; code
+// signing is unsupported, so it only proves the function exists.
+func (d Deps) getFunctionCodeSigningConfig(ctx context.Context, c *call) (reply, error) {
+	if _, _, err := d.lookup(ctx, c, ""); err != nil {
+		return reply{}, err
+	}
+	return reply{Status: http.StatusOK, Body: noCodeSigning}, nil
+}
+
+// getFunction implements GetFunction for the $LATEST version.
+func (d Deps) getFunction(ctx context.Context, c *call) (reply, error) {
+	rec, cfg, err := d.loadConfiguration(ctx, c)
+	if err != nil {
+		return reply{}, err
 	}
 	out := &lambda.GetFunctionOutput{
 		Configuration: cfg,

spinifex/domains/lambda/awsapi/handler_test.go

@@ -662,8 +662,211 @@ func TestAuthorization(t *testing.T) {
 
 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateFunction", "GetFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration"}, reg.Inventory.Registered)
+	assert.Equal(t, "GetFunctionCodeSigningConfig", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2020-06-30/functions/x/code-signing-config", nil)))
 	assert.Equal(t, "CreateFunction", reg.ResolveAction(httptest.NewRequest(http.MethodPost, "/2015-03-31/functions", nil)))
 	assert.Equal(t, "GetFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x", nil)))
+	assert.Equal(t, "GetFunctionConfiguration", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/configuration", nil)))
 	assert.Empty(t, reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x", nil)))
 }
+
+func TestGetFunctionConfiguration_ReturnsTheSameConfigurationAsGetFunction(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+
+	rec, err := e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+	require.NoError(t, err)
+	var full lambda.GetFunctionOutput
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &full))
+
+	rec, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, rec.Code)
+	assert.NotContains(t, rec.Body.String(), `"Code"`, "the configuration is the top-level body, not GetFunction's wrapper")
+	var cfg lambda.FunctionConfiguration
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &cfg))
+	assert.Equal(t, *full.Configuration, cfg)
+	assert.Equal(t, "prod", *cfg.Environment.Variables["MODE"])
+
+	arn := "arn%3Aaws%3Alambda%3A" + gwtest.Region + "%3A" + gwtest.AccountID + "%3Afunction%3Ahello"
+	for _, path := range []string{
+		"/2015-03-31/functions/hello/configuration?Qualifier=%24LATEST",
+		"/2015-03-31/functions/" + arn + "/configuration",
+		"/2015-03-31/functions/" + arn + "%3A%24LATEST/configuration",
+	} {
+		rec, err := e.do(http.MethodGet, path, nil)
+		require.NoError(t, err, path)
+		assert.Equal(t, http.StatusOK, rec.Code, path)
+	}
+}
+
+func TestGetFunctionConfiguration_MissingAndUnsupportedMatchGetFunction(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	for label, tc := range map[string]struct{ suffix, code string }{
+		"missing":         {"ghost", awserrors.ErrorResourceNotFoundException},
+		"version":         {"hello?Qualifier=3", awserrors.ErrorResourceNotFoundException},
+		"other account":   {"arn%3Aaws%3Alambda%3A" + gwtest.Region + "%3A999999999999%3Afunction%3Ahello", awserrors.ErrorResourceNotFoundException},
+		"qualifier clash": {"hello%3A%24LATEST?Qualifier=3", awserrors.ErrorInvalidParameterValueException},
+		"not lambda":      {"arn%3Aaws%3As3%3A%3A%3Abucket", awserrors.ErrorInvalidParameterValueException},
+	} {
+		t.Run(label, func(t *testing.T) {
+			path, query, _ := strings.Cut(tc.suffix, "?")
+			if query != "" {
+				query = "?" + query
+			}
+			_, getErr := e.do(http.MethodGet, "/2015-03-31/functions/"+path+query, nil)
+			requireCode(t, getErr, tc.code)
+			_, err := e.do(http.MethodGet, "/2015-03-31/functions/"+path+"/configuration"+query, nil)
+			requireCode(t, err, tc.code)
+			assert.Equal(t, getErr.Error(), err.Error())
+		})
+	}
+
+	e.h.Region = "us-west-2"
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestGetFunctionConfiguration_DoesNotNeedTheCodeStore(t *testing.T) {
+	for name, codes := range map[string]awsapi.CodeStoreFunc{
+		"store outage": func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") },
+		"no store":     nil,
+	} {
+		t.Run(name, func(t *testing.T) {
+			e := newEnv(t)
+			_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+			require.NoError(t, err)
+			e.deps.Codes = codes
+
+			rec, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+			require.NoError(t, err)
+			require.Equal(t, http.StatusOK, rec.Code)
+			var cfg lambda.FunctionConfiguration
+			require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &cfg))
+			assert.Equal(t, "hello", *cfg.FunctionName)
+
+			_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+			requireCode(t, err, awserrors.ErrorServiceException)
+		})
+	}
+}
+
+func TestGetFunctionConfiguration_Authorization(t *testing.T) {
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	setup := func(t *testing.T, statements ...iamdomain.Statement) *env {
+		e := newEnv(t, statements...)
+		_, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+			FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+			Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+		})
+		require.NoError(t, err)
+		return e
+	}
+	get := func(e *env) error {
+		_, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+		return err
+	}
+
+	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))))
+	})
+	t.Run("a deny on the function ARN applies", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Deny", "lambda:GetFunctionConfiguration", arn))))
+	})
+	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionConfiguration", arn+"-other"))))
+	})
+	t.Run("an allow on the function ARN passes", func(t *testing.T) {
+		require.NoError(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionConfiguration", arn))))
+	})
+}
+
+func TestGetFunctionCodeSigningConfig_ExistingFunctionHasExplicitNulls(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	arn := "arn%3Aaws%3Alambda%3A" + gwtest.Region + "%3A" + gwtest.AccountID + "%3Afunction%3Ahello"
+	for _, name := range []string{"hello", arn, gwtest.AccountID + "%3Afunction%3Ahello"} {
+		rec, err := e.do(http.MethodGet, "/2020-06-30/functions/"+name+"/code-signing-config", nil)
+		require.NoError(t, err, name)
+		require.Equal(t, http.StatusOK, rec.Code, name)
+		assert.Equal(t, "application/json", rec.Header().Get("Content-Type"))
+		assert.JSONEq(t, `{"CodeSigningConfigArn":null,"FunctionName":null}`, rec.Body.String(), name)
+	}
+}
+
+func TestGetFunctionCodeSigningConfig_MissingFunctionIsNotFound(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	_, err = e.do(http.MethodGet, "/2020-06-30/functions/ghost/code-signing-config", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, getErr := e.do(http.MethodGet, "/2015-03-31/functions/ghost", nil)
+	assert.Equal(t, getErr.Error(), err.Error())
+
+	e.h.Region = "us-west-2"
+	_, err = e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestGetFunctionCodeSigningConfig_NeedsNeitherCodeStoreNorDecryption(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+	e.deps.Codes = func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") }
+	e.deps.AEAD = nil
+
+	rec, err := e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{"CodeSigningConfigArn":null,"FunctionName":null}`, rec.Body.String())
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	requireCode(t, err, awserrors.ErrorServiceException)
+
+	e.deps.Codes = nil
+	_, err = e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+	require.NoError(t, err)
+}
+
+func TestGetFunctionCodeSigningConfig_Authorization(t *testing.T) {
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	setup := func(t *testing.T, statements ...iamdomain.Statement) *env {
+		e := newEnv(t, statements...)
+		_, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+			FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+			Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+		})
+		require.NoError(t, err)
+		return e
+	}
+	get := func(e *env) error {
+		_, err := e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+		return err
+	}
+
+	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))))
+	})
+	t.Run("a deny on the function ARN applies", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Deny", "lambda:GetFunctionCodeSigningConfig", arn))))
+	})
+	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionCodeSigningConfig", arn+"-other"))))
+	})
+	t.Run("an allow on the function ARN passes", func(t *testing.T) {
+		require.NoError(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionCodeSigningConfig", arn))))
+	})
+}

spinifex/domains/lambda/awsapi/registration.go

@@ -117,7 +117,7 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		if decodeErr == nil {
 			resource = functionARN(inv.Region, inv.AccountID, aws.StringValue(c.Create.FunctionName))
 		}
-	case "GetFunction":
+	case "GetFunction", "GetFunctionConfiguration", "GetFunctionCodeSigningConfig":
 		if name, _, err := resolveFunctionName(inv.Region, inv.AccountID, params[0]); err == nil {
 			resource = functionARN(inv.Region, inv.AccountID, name)
 		}
@@ -133,7 +133,9 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 			return err
 		}
 	}
-	if d.Meta == nil || d.Codes == nil {
+	// Only GetFunction presigns code; the other reads are metadata only.
+	needsCodes := action != "GetFunctionConfiguration" && action != "GetFunctionCodeSigningConfig"
+	if d.Meta == nil || (d.Codes == nil && needsCodes) {
 		return errors.New(awserrors.ErrorServiceException)
 	}
 
@@ -180,7 +182,13 @@ func (d Deps) checkPassRole(inv dispatch.Invocation, roleARN string) error {
 }
 
 func writeJSON(w http.ResponseWriter, out reply) {
-	body, err := jsonutil.BuildJSON(out.Body)
+	var body []byte
+	var err error
+	if raw, ok := out.Body.(rawJSON); ok {
+		body = raw
+	} else {
+		body, err = jsonutil.BuildJSON(out.Body)
+	}
 	if err != nil {
 		slog.Error("Failed to marshal Lambda response", "err", err)
 		w.Header().Set("Content-Type", "application/json")

spinifex/domains/lambda/awsapi/routes.go

@@ -21,6 +21,10 @@ type call struct {
 	Create *lambda.CreateFunctionInput
 }
 
+// rawJSON is a response body written as is, for a shape the SDK encoder
+// cannot produce.
+type rawJSON []byte
+
 // reply is a handler's success: the HTTP status and the SDK shape to encode.
 type reply struct {
 	Status int
@@ -41,6 +45,14 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.getFunction(ctx, c)
 		}},
+	{Method: "GET", Pattern: "/2015-03-31/functions/{FunctionName}/configuration", Action: "GetFunctionConfiguration",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.getFunctionConfiguration(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2020-06-30/functions/{FunctionName}/code-signing-config", Action: "GetFunctionCodeSigningConfig",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.getFunctionCodeSigningConfig(ctx, c)
+		}},
 }
 
 // actionNames returns the distinct actions in routes in stable order.