feat(ui): console feedback v1 and the admin Nodes view (#1454)

* feat(ui): console feedback round: state tones, inline tags, filter bar and form fixes

Settled states go green and transitions yellow with a spinner, status text is
lowercase, tags are edited inline row by row, every list filters through one
filter bar, long cells truncate with a tooltip and times are relative.

EC2 launch takes a Name and offers ICMP echo, and leaves Placement Group out
when no group is available. RDS create is reordered with engine tiles and
window pickers. Route 53 records get bulk delete, a TTL select, value labels
that follow the type, and NS. Tooltips use the popover tokens, and the console
is sized over a 110% root.

* feat(ui): size RDS instance classes and give each record value its own input

An RDS class is a db.-prefixed EC2 type, so the class picker shows its vCPUs
and memory in every option and under the field, as the EC2 launch form does,
on create, modify and restore.

A Route 53 record's values were one textarea, which let a line break into a
value. Each value is now a single-line input with Add value; Enter starts the
next one, a pasted list splits into rows, and CNAME takes exactly one.

* fix(ui): open a DNS record for editing from the keyboard

* feat: admin node inventory, topology, live host metrics and network test

Each node collects its hardware facts as root at daemon start (dmidecode,
ethtool, networkctl LLDP where available) into /etc/spinifex/node-facts, and
the daemon serves them with its plane wiring over NATS.

New admin-only actions, refused for every account but the system admin
account regardless of policy:

- GetNodeFacts: hardware, NICs, LLDP neighbours and planes per node
- GetNodeMetrics: live CPU, memory and per-plane traffic for one node
- StartNetworkTest / DescribeNetworkTests: iperf2 between nodes over a
  chosen plane, both directions, one test cluster-wide at a time, recorded
  in KV and converged asynchronously

The iperf listener runs only for the duration of a run, without
capabilities, on TCP 5001, which the peers firewall rule now allows. A run
refuses any target outside the node's own plane subnet.

The console gains a node panel (hardware, network, metrics, network test
with a dial scaled to the port's line rate) and a topology view built with
@xyflow/react. setup.sh installs dmidecode and iperf.

* test: cover the node insight handlers and the facts file mode

* feat: report every CPU flag in node facts

* feat(ui): node table on the dashboard, every CPU flag and a readable topology

The dashboard's nodes section is the Nodes page table, open by default, and a
row opens that node. Ready reads green, capacity sorts by most available, and
the processor's features tip lists every flag. Topology draws one row per
port with switches in a muted card, highlights a node's links on hover, and
offers a left-to-right layout beside top-down.

* chore(ui): rebuild dist

* fix: align the CPU facts fields

* fix: bound the network test server start and read ARM CPU features and cores

* chore(ui): rebuild dist

* test(ui): pin the instances baseline clock and take the screenshots from CI's Playwright image

* fix: retry a network test claim whose holder released it mid-check

.gitignore

@@ -24,6 +24,9 @@ build/microvm/qboot.LICENSE
 build/microvm/sqs-worker/vmlinuz
 build/microvm/sqs-worker/initramfs.cpio.gz
 build/microvm/sqs-worker/.stage.*/
+build/microvm/lambda-runtime/vmlinuz
+build/microvm/lambda-runtime/initramfs.cpio.gz
+build/microvm/lambda-runtime/.stage.*/
 build/qboot/
 !spinifex/runtime/roles/spinifexui/frontend/dist/
 tmp/

Makefile

@@ -169,7 +169,7 @@ go_local_srcs = $(shell GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go list
 # worker as separate processes. A separate artifact from the LB image above.
 SQS_WORKER_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/sqs-worker
 SQS_WORKER_MICROVM_ARTIFACTS := $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz $(SQS_WORKER_MICROVM_OUT_DIR)/initramfs.cpio.gz
-SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_WORKER_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-store sqs-worker-ready sqs-worker-stop) bin/nats-server bin/sqs-worker
+SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_WORKER_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-store sqs-worker-ready sqs-worker-stop) bin/nats-server bin/sqs-worker
 
 $(SQS_WORKER_MICROVM_ARTIFACTS) &: $(SQS_WORKER_MICROVM_INPUTS)
 	./scripts/build-sqs-store-microvm-image.sh
@@ -180,6 +180,20 @@ bin/sqs-worker: go.mod go.sum $$(call go_local_srcs,./cmd/sqs-worker)
 
 build-sqs-worker-microvm-image: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Build the SQS microVM kernel and initramfs
 
+# The Lambda runtime microVM image. Built on demand; not shipped or installed.
+LAMBDA_RUNTIME_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/lambda-runtime
+LAMBDA_RUNTIME_MICROVM_ARTIFACTS := $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/vmlinuz $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/initramfs.cpio.gz
+LAMBDA_RUNTIME_MICROVM_INPUTS := scripts/build-lambda-runtime-microvm-image.sh scripts/lib/microvm-image.sh $(addprefix $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/,init.sh inittab run-agent) bin/lambda-agent
+
+$(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) &: $(LAMBDA_RUNTIME_MICROVM_INPUTS)
+	./scripts/build-lambda-runtime-microvm-image.sh
+
+bin/lambda-agent: go.mod go.sum $$(call go_local_srcs,./cmd/lambda-agent)
+	@mkdir -p bin
+	GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/lambda-agent
+
+build-lambda-runtime-microvm-image: $(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) ## Build the Lambda runtime microVM kernel and initramfs (not shipped)
+
 # qboot is the microvm firmware. The source tarball ships with each release
 # because qboot is GPL-2.0.
 QBOOT_COMMIT := $(shell sed -n 's/^QBOOT_COMMIT="\(.*\)"$$/\1/p' scripts/build-qboot.sh)
@@ -522,7 +536,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
+.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

build/microvm/lambda-runtime/init.sh

@@ -0,0 +1,68 @@
+#!/bin/sh
+mount -t proc  proc  /proc
+mount -t sysfs sys   /sys
+mount -t devtmpfs dev /dev
+
+# modprobe needs a usable modules.dep; insmod by explicit path does not, so
+# fall back to it. Handles .ko and .ko.gz.
+load_mod() {
+    modprobe "$1" 2>/dev/null && return 0
+    for ko in $(find /lib/modules -name "$1.ko" -o -name "$1.ko.gz" 2>/dev/null); do
+        insmod "$ko" 2>/dev/null && return 0
+        case "$ko" in
+            *.gz) gunzip -c "$ko" >/tmp/mod.ko 2>/dev/null \
+                  && insmod /tmp/mod.ko 2>/dev/null && return 0 ;;
+        esac
+    done
+    return 1
+}
+# lambda-agent reads its identity from fw_cfg and unloads qemu_fw_cfg before
+# anything else runs. virtio_net needs net_failover and failover, and the
+# insmod fallback resolves no dependencies, so load the chain bottom-up.
+load_mod qemu_fw_cfg || echo "[init] qemu_fw_cfg did not load" >/dev/console
+load_mod failover || true
+load_mod net_failover || true
+load_mod virtio_net || echo "[init] virtio_net did not load" >/dev/console
+
+ip link set lo up
+
+# The guest's addresses come from the same netcfg entry as other system VMs.
+NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
+i=0
+while [ "$i" -lt 25 ] && [ ! -f "$NETCFG" ]; do
+    sleep 0.04
+    i=$((i + 1))
+done
+if [ -f "$NETCFG" ]; then
+    # shellcheck source=/dev/null
+    . "$NETCFG"
+    for n in 0 1 2 3 4 5; do
+        eval mac=\$NIC${n}_MAC
+        [ -z "$mac" ] && continue
+        iface=""
+        for _ in 1 2 3 4 5; do
+            iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :)
+            [ -n "$iface" ] && break
+            sleep 0.02
+        done
+        if [ -z "$iface" ]; then
+            echo "[init] no interface for MAC $mac (NIC$n)" >/dev/console
+            continue
+        fi
+        cidr="" gw="" default="" rdst="" rvia=""
+        eval cidr=\$NIC${n}_CIDR
+        eval gw=\$NIC${n}_GW
+        eval default=\$NIC${n}_DEFAULT
+        eval rdst=\$NIC${n}_ROUTE_DST
+        eval rvia=\$NIC${n}_ROUTE_VIA
+        ip link set "$iface" up
+        [ -n "$cidr" ] && ip addr add "$cidr" dev "$iface"
+        [ "$default" = "1" ] && [ -n "$gw" ] && ip route add default via "$gw" dev "$iface"
+        [ -n "$rdst" ] && [ -n "$rvia" ] && ip route add "$rdst" via "$rvia" dev "$iface"
+    done
+else
+    # lambda-agent then fails to reach the host and the guest powers off.
+    echo "[init] fw_cfg netcfg not available" >/dev/console
+fi
+
+exec /sbin/init

build/microvm/lambda-runtime/inittab

@@ -0,0 +1,6 @@
+# The power button loads here, not in /init: busybox init has its SIGUSR2
+# (poweroff) handler installed before sysinit runs, so a press can never reach
+# the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
+::sysinit:/sbin/modprobe tiny_power_button power_signal=12
+::once:/usr/local/lib/lambda-runtime/run-agent
+::ctrlaltdel:/sbin/reboot

build/microvm/lambda-runtime/run-agent

@@ -0,0 +1,6 @@
+#!/bin/sh
+# Runs the Lambda guest agent, then powers the guest off whatever its status.
+/usr/local/bin/lambda-agent >/dev/console 2>&1
+status=$?
+echo "[run-agent] lambda-agent exited with status $status" >/dev/console
+poweroff -f

build/systemd/spinifex-daemon.service

@@ -1,4 +1,4 @@
-# spinifex-unit-version: 5
+# spinifex-unit-version: 6
 [Unit]
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
@@ -15,6 +15,7 @@ Slice=spinifex-system.slice
 User=spinifex-daemon
 Group=spinifex
 ExecStartPre=/usr/local/bin/spx admin hostnet check
+ExecStartPre=-+/usr/local/bin/spx admin node-facts collect
 ExecStart=/usr/local/bin/spx service spinifex start
 Restart=on-failure
 RestartSec=5

cmd/lambda-agent/main.go

@@ -0,0 +1,24 @@
+// lambda-agent runs inside a Lambda guest.
+//
+// A normal build reads its per-generation identity from fw_cfg, unloads
+// fw_cfg so nothing started later can read the guest key, and opens the
+// authenticated control connection to the host. It accepts no invocations and
+// never starts the function's bootstrap: the message protocol and grant
+// fencing do not exist yet. The agent never listens for the host.
+//
+// A lambdatestlink build instead relays invocations over an unauthenticated
+// test carrier for local proofs; it is not the guest protocol.
+package main
+
+import (
+	"context"
+	"flag"
+	"os"
+)
+
+func main() { os.Exit(run()) }
+
+func run() int {
+	flag.Parse()
+	return start(context.Background())
+}

cmd/lambda-agent/start_tcp.go

@@ -0,0 +1,84 @@
+//go:build !lambdatestlink
+
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+)
+
+// Exit statuses: the guest refused to start, the control connection failed,
+// or the host closed it.
+const (
+	exitRefused = 2
+	exitLink    = 1
+	exitClosed  = 0
+)
+
+// errNoProtocol is returned when the host sends anything: this build defines no
+// message protocol, so it accepts no work.
+var errNoProtocol = errors.New("host sent data, but this build accepts no invocations")
+
+type startup struct {
+	fwCfgDir  string
+	fwCfgRoot string
+	unload    func() error
+}
+
+func start(ctx context.Context) int {
+	s := startup{fwCfgDir: hostlink.FwCfgDir, fwCfgRoot: hostlink.FwCfgRoot, unload: hostlink.UnloadFwCfg}
+	code, err := s.run(ctx)
+	if err != nil {
+		slog.Error("lambda-agent stopping", "exit", code, "err", err)
+	}
+	return code
+}
+
+// run loads the identity, seals fw_cfg whether or not the load succeeded,
+// then authenticates to the host and holds the connection until it closes.
+func (s startup) run(ctx context.Context) (int, error) {
+	id, loadErr := hostlink.LoadIdentity(s.fwCfgDir)
+	if err := hostlink.SealFwCfg(s.fwCfgRoot, s.unload); err != nil {
+		return exitRefused, errors.Join(loadErr, err)
+	}
+	if loadErr != nil {
+		return exitRefused, loadErr
+	}
+	conn, err := hostlink.Dial(ctx, id)
+	if err != nil {
+		if ctx.Err() != nil {
+			err = fmt.Errorf("%w: %w", err, context.Cause(ctx))
+		}
+		return exitLink, err
+	}
+	defer conn.Close()
+	slog.Info("control connection open; invocations are not enabled in this build", "host", id.HostAddr)
+	if err := awaitHostClose(ctx, conn); err != nil {
+		return exitLink, err
+	}
+	return exitClosed, nil
+}
+
+// awaitHostClose reads until the host closes the connection. Under TLS 1.3 a
+// host that rejects the guest certificate is only seen here, on first read.
+func awaitHostClose(ctx context.Context, conn *tls.Conn) error {
+	stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
+	defer stop()
+	n, err := conn.Read(make([]byte, 1))
+	switch {
+	case n > 0:
+		return errNoProtocol
+	case errors.Is(err, io.EOF):
+		return nil
+	case ctx.Err() != nil:
+		return fmt.Errorf("control connection: %w", context.Cause(ctx))
+	default:
+		return fmt.Errorf("control connection: %w", err)
+	}
+}

cmd/lambda-agent/start_tcp_test.go

@@ -0,0 +1,256 @@
+//go:build !lambdatestlink
+
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/guestconn"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-1"
+
+// guest is a fake fw_cfg tree; unload removes it as unloading the module would.
+type guest struct {
+	root     string
+	unloads  int
+	sealedAt chan struct{}
+}
+
+func newGuest(t *testing.T, entries map[string][]byte) *guest {
+	t.Helper()
+	g := &guest{root: filepath.Join(t.TempDir(), "qemu_fw_cfg"), sealedAt: make(chan struct{}, 1)}
+	for name, b := range entries {
+		dir := filepath.Join(g.root, "by_name", lambdav1.FwCfgPrefix, name)
+		require.NoError(t, os.MkdirAll(dir, 0o700))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "raw"), b, 0o600))
+	}
+	return g
+}
+
+func (g *guest) startup() startup {
+	return startup{
+		fwCfgDir:  filepath.Join(g.root, "by_name", lambdav1.FwCfgPrefix),
+		fwCfgRoot: g.root,
+		unload: func() error {
+			g.unloads++
+			g.sealedAt <- struct{}{}
+			return os.RemoveAll(g.root)
+		},
+	}
+}
+
+type host struct {
+	ln   *net.TCPListener
+	gen  *guestconn.Generation
+	conn chan accepted
+}
+
+func newHost(t *testing.T) *host {
+	t.Helper()
+	ln, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)})
+	require.NoError(t, err)
+	t.Cleanup(func() { ln.Close() })
+	gen, err := guestconn.NewGeneration(scope, time.Hour)
+	require.NoError(t, err)
+	return &host{ln: ln, gen: gen, conn: make(chan accepted, 1)}
+}
+
+func (h *host) entries() map[string][]byte { return h.gen.GuestEntries(h.ln.Addr().String()) }
+
+type accepted struct {
+	conn *tls.Conn
+	err  error
+}
+
+// accept runs the host end until ctx ends and reports what it accepted.
+func (h *host) accept(ctx context.Context) {
+	go func() {
+		c, err := h.gen.AcceptGuest(ctx, h.ln, 2*time.Second)
+		h.conn <- accepted{c, err}
+	}()
+}
+
+// guest waits for the host to authenticate the guest.
+func (h *host) guest(t *testing.T) *tls.Conn {
+	t.Helper()
+	a := wait(t, h.conn, "host to authenticate the guest")
+	require.NoError(t, a.err)
+	return a.conn
+}
+
+// wait bounds every test wait so a stalled host or startup fails promptly.
+func wait[T any](t *testing.T, ch <-chan T, what string) T {
+	t.Helper()
+	select {
+	case v := <-ch:
+		return v
+	case <-time.After(10 * time.Second):
+		t.Fatalf("timed out waiting for %s", what)
+		panic("unreachable")
+	}
+}
+
+type result struct {
+	code int
+	err  error
+}
+
+func runAsync(ctx context.Context, s startup) <-chan result {
+	out := make(chan result, 1)
+	go func() {
+		code, err := s.run(ctx)
+		out <- result{code, err}
+	}()
+	return out
+}
+
+func TestStartupAuthenticatesAfterSealingFwCfg(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	g := newGuest(t, h.entries())
+	done := runAsync(ctx, g.startup())
+
+	conn := h.guest(t)
+	select {
+	case <-g.sealedAt:
+	default:
+		t.Fatal("guest dialled before fw_cfg was sealed")
+	}
+	_, err := os.Stat(g.root)
+	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg still readable while connected")
+
+	require.NoError(t, conn.Close())
+	res := wait(t, done, "startup to return")
+	assert.Equal(t, exitClosed, res.code)
+	assert.NoError(t, res.err)
+	assert.Equal(t, 1, g.unloads)
+}
+
+func TestStartupRefusesHostData(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	done := runAsync(ctx, newGuest(t, h.entries()).startup())
+
+	conn := h.guest(t)
+	defer conn.Close()
+	_, err := conn.Write([]byte{0})
+	require.NoError(t, err)
+	res := wait(t, done, "startup to return")
+	assert.Equal(t, exitLink, res.code)
+	assert.ErrorIs(t, res.err, errNoProtocol)
+}
+
+// The host pins another generation's guest; the guest learns of the refusal
+// on its first read and fails with a link error, never a clean close.
+func TestStartupFailsWhenHostRefusesGuest(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	other, err := guestconn.NewGeneration(scope, time.Hour)
+	require.NoError(t, err)
+	entries := h.entries()
+	wrong := other.GuestEntries(h.ln.Addr().String())
+	entries[lambdav1.EntryGuestKey], entries[lambdav1.EntryGuestCert] = wrong[lambdav1.EntryGuestKey], wrong[lambdav1.EntryGuestCert]
+	g := newGuest(t, entries)
+
+	res := wait(t, runAsync(ctx, g.startup()), "startup to return")
+	assert.Equal(t, exitLink, res.code)
+	require.Error(t, res.err)
+	assert.Contains(t, res.err.Error(), "bad certificate")
+	assert.Equal(t, 1, g.unloads)
+	select {
+	case a := <-h.conn:
+		require.Error(t, a.err, "host accepted a guest it does not pin")
+	default:
+	}
+}
+
+func TestStartupRefusesWithoutDialling(t *testing.T) {
+	h := newHost(t)
+	cases := map[string]struct {
+		entries map[string][]byte
+		unload  func(g *guest) func() error
+		want    string
+	}{
+		"missing entry": {
+			entries: func() map[string][]byte { e := h.entries(); delete(e, lambdav1.EntryHostCert); return e }(),
+			want:    "read host-cert",
+		},
+		"fw_cfg stays readable": {
+			entries: h.entries(),
+			unload:  func(*guest) func() error { return func() error { return errors.New("module is built in") } },
+			want:    "fw_cfg still readable",
+		},
+		"unload claims success but tree remains": {
+			entries: h.entries(),
+			unload:  func(*guest) func() error { return func() error { return nil } },
+			want:    "unload reported success",
+		},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			g := newGuest(t, tc.entries)
+			s := g.startup()
+			if tc.unload != nil {
+				s.unload = tc.unload(g)
+			}
+			code, err := s.run(context.Background())
+			assert.Equal(t, exitRefused, code)
+			assert.ErrorContains(t, err, tc.want)
+		})
+	}
+
+	// Nothing dialled the host in any case above.
+	require.NoError(t, h.ln.SetDeadline(time.Now().Add(50*time.Millisecond)))
+	_, err := h.ln.Accept()
+	var ne net.Error
+	require.ErrorAs(t, err, &ne)
+	assert.True(t, ne.Timeout())
+}
+
+// A missing entry still seals fw_cfg: the key may be present even when
+// another entry is not.
+func TestStartupSealsEvenWhenIdentityIsIncomplete(t *testing.T) {
+	h := newHost(t)
+	e := h.entries()
+	delete(e, lambdav1.EntryHostAddr)
+	g := newGuest(t, e)
+	code, err := g.startup().run(context.Background())
+	assert.Equal(t, exitRefused, code)
+	assert.ErrorContains(t, err, "read host-addr")
+	assert.Equal(t, 1, g.unloads)
+	_, statErr := os.Stat(g.root)
+	assert.ErrorIs(t, statErr, os.ErrNotExist)
+}
+
+func TestStartupStopsWithContext(t *testing.T) {
+	ctx, cancel := context.WithCancelCause(context.Background())
+	defer cancel(nil)
+	h := newHost(t)
+	h.accept(ctx)
+	done := runAsync(ctx, newGuest(t, h.entries()).startup())
+	conn := h.guest(t)
+	defer conn.Close()
+
+	reason := errors.New("guest shutting down")
+	cancel(reason)
+	res := wait(t, done, "startup to return")
+	assert.Equal(t, exitLink, res.code)
+	assert.ErrorIs(t, res.err, reason)
+}

cmd/lambda-agent/start_testlink.go

@@ -0,0 +1,87 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+)
+
+const (
+	runtimeAddr = "127.0.0.1:9001"
+	taskRoot    = "/var/task"
+)
+
+var (
+	linkPath = flag.String("link", os.Getenv("LAMBDA_AGENT_LINK"), "test carrier device path")
+	handler  = flag.String("handler", os.Getenv("_HANDLER"), "function handler passed to the bootstrap")
+)
+
+func start(parent context.Context) int {
+	if *linkPath == "" {
+		slog.Error("no test carrier configured; refusing to start")
+		return 2
+	}
+	// The bootstrap is tenant code; fw_cfg must be unreadable before it runs.
+	if err := hostlink.SealFwCfg(hostlink.FwCfgRoot, hostlink.UnloadFwCfg); err != nil {
+		slog.Error("seal fw_cfg; refusing to start", "err", err)
+		return 2
+	}
+	dev, err := os.OpenFile(*linkPath, os.O_RDWR, 0)
+	if err != nil {
+		slog.Error("open test carrier; refusing to start", "err", err)
+		return 2
+	}
+	defer dev.Close()
+	link := hostlink.NewTestStream(dev)
+
+	srv := runtimeapi.New()
+	ln, err := net.Listen("tcp", runtimeAddr)
+	if err != nil {
+		slog.Error("listen runtime api", "err", err)
+		return 1
+	}
+	hs := &http.Server{Handler: srv.Handler(), ReadHeaderTimeout: 10 * time.Second}
+	go func() { _ = hs.Serve(ln) }()
+	defer hs.Close()
+
+	ctx, cancel := context.WithCancelCause(parent)
+	defer cancel(nil)
+
+	cmd, err := agent.StartBootstrap(ctx, taskRoot+"/bootstrap", []string{
+		"AWS_LAMBDA_RUNTIME_API=" + runtimeAddr,
+		"_HANDLER=" + *handler,
+		"LAMBDA_TASK_ROOT=" + taskRoot,
+		"PATH=/bin:/sbin:/usr/bin",
+	}, os.Stdout, os.Stderr)
+	if err != nil {
+		slog.Error("start bootstrap", "err", err)
+		return 1
+	}
+	exited := make(chan struct{})
+	go func() {
+		defer close(exited)
+		err := cmd.Wait()
+		cancel(fmt.Errorf("bootstrap exited: %w", err))
+	}()
+
+	err = agent.Run(ctx, link, srv)
+	cancel(errors.New("agent stopping"))
+	<-exited
+	if err != nil {
+		slog.Error("agent stopped", "err", err)
+		return 1
+	}
+	return 0
+}

contracts/cluster/v1/facts.go

@@ -0,0 +1,211 @@
+package clusterv1
+
+import "regexp"
+
+const (
+	// NodeFactsSubject fans out a request for each daemon's hardware and
+	// cabling description. Its reply payload is NodeFacts.
+	NodeFactsSubject = "spinifex.node.facts"
+
+	nodeNetTestPrefix = "spinifex.node.nettest."
+)
+
+var nodeIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
+
+// ValidNodeID reports whether a node name is safe to place in a subject: no
+// NATS token separator or wildcard can get through.
+func ValidNodeID(node string) bool {
+	return nodeIDPattern.MatchString(node)
+}
+
+// NetTestServeSubject asks one node to start a one-shot throughput server.
+func NetTestServeSubject(node string) string {
+	return nodeNetTestPrefix + node + ".serve"
+}
+
+// NetTestRunSubject asks one node to run the client side of a throughput test.
+func NetTestRunSubject(node string) string {
+	return nodeNetTestPrefix + node + ".run"
+}
+
+// Planes a node's NICs can carry, named as the installer names them.
+const (
+	PlaneWAN = "wan"
+	PlaneLAN = "lan"
+	PlaneVPC = "vpc"
+)
+
+// NetTestPort is the TCP port reserved for node-to-node throughput tests.
+const NetTestPort = 5001
+
+// NodeFacts describes one node's hardware, OS and cabling. Fields a host
+// cannot report are left empty, and Sources says why.
+type NodeFacts struct {
+	Node        string            `json:"node"`
+	CollectedAt string            `json:"collected_at,omitempty"`
+	FormFactor  string            `json:"form_factor"`
+	Virt        string            `json:"virt,omitempty"`
+	System      SystemFacts       `json:"system"`
+	OS          OSFacts           `json:"os"`
+	CPU         CPUFacts          `json:"cpu"`
+	Memory      MemoryFacts       `json:"memory"`
+	NICs        []NICFacts        `json:"nics"`
+	Planes      []PlaneFacts      `json:"planes"`
+	Sources     map[string]string `json:"sources"`
+}
+
+// SystemFacts is the machine's identity from DMI.
+type SystemFacts struct {
+	Vendor       string `json:"vendor,omitempty"`
+	Product      string `json:"product,omitempty"`
+	Family       string `json:"family,omitempty"`
+	Version      string `json:"version,omitempty"`
+	Serial       string `json:"serial,omitempty"`
+	UUID         string `json:"uuid,omitempty"`
+	ChassisType  string `json:"chassis_type,omitempty"`
+	ChassisAsset string `json:"chassis_asset,omitempty"`
+	BoardVendor  string `json:"board_vendor,omitempty"`
+	BoardProduct string `json:"board_product,omitempty"`
+	BoardSerial  string `json:"board_serial,omitempty"`
+	BIOSVendor   string `json:"bios_vendor,omitempty"`
+	BIOSVersion  string `json:"bios_version,omitempty"`
+	BIOSDate     string `json:"bios_date,omitempty"`
+	DeviceModel  string `json:"device_model,omitempty"`
+}
+
+// OSFacts names the running OS and Spinifex build.
+type OSFacts struct {
+	Name          string `json:"name,omitempty"`
+	Kernel        string `json:"kernel,omitempty"`
+	Arch          string `json:"arch,omitempty"`
+	Version       string `json:"spx_version,omitempty"`
+	Commit        string `json:"spx_commit,omitempty"`
+	UptimeSeconds int64  `json:"uptime_seconds,omitempty"`
+	Hostname      string `json:"hostname,omitempty"`
+}
+
+// CPUFacts summarises the sockets. Threads and Cores are totals. Features are
+// the flags worth naming; Flags is every flag the CPU reports.
+type CPUFacts struct {
+	Model          string   `json:"model,omitempty"`
+	Vendor         string   `json:"vendor,omitempty"`
+	Sockets        int      `json:"sockets"`
+	CoresPerSocket int      `json:"cores_per_socket,omitempty"`
+	Cores          int      `json:"cores"`
+	Threads        int      `json:"threads"`
+	MaxMHz         int      `json:"max_mhz,omitempty"`
+	CurrentMHz     int      `json:"current_mhz,omitempty"`
+	Virtualization string   `json:"virtualization,omitempty"`
+	Features       []string `json:"features,omitempty"`
+	Flags          []string `json:"flags,omitempty"`
+}
+
+// MemoryFacts groups identical DIMMs so a 32-slot board reads as one line.
+type MemoryFacts struct {
+	TotalBytes    uint64        `json:"total_bytes"`
+	SlotsUsed     int           `json:"slots_used"`
+	SlotsTotal    int           `json:"slots_total"`
+	MaxCapacity   string        `json:"max_capacity,omitempty"`
+	ECC           string        `json:"ecc,omitempty"`
+	Groups        []MemoryGroup `json:"groups"`
+	OSTotalBytes  uint64        `json:"os_total_bytes,omitempty"`
+	OSAvailBytes  uint64        `json:"os_available_bytes,omitempty"`
+	HugePageBytes uint64        `json:"huge_page_bytes,omitempty"`
+}
+
+// MemoryGroup is a set of DIMMs alike in every field shown. ConfiguredMTs
+// below SpeedMTs means the board runs them slower than rated.
+type MemoryGroup struct {
+	Count         int      `json:"count"`
+	SizeBytes     uint64   `json:"size_bytes"`
+	Type          string   `json:"type,omitempty"`
+	TypeDetail    string   `json:"type_detail,omitempty"`
+	FormFactor    string   `json:"form_factor,omitempty"`
+	SpeedMTs      int      `json:"speed_mts,omitempty"`
+	ConfiguredMTs int      `json:"configured_mts,omitempty"`
+	Manufacturer  string   `json:"manufacturer,omitempty"`
+	PartNumber    string   `json:"part_number,omitempty"`
+	Rank          string   `json:"rank,omitempty"`
+	Serials       []string `json:"serials,omitempty"`
+}
+
+// NICFacts is one physical network interface and, where LLDP reports it, the
+// switch port it is cabled to.
+type NICFacts struct {
+	Name      string        `json:"name"`
+	MAC       string        `json:"mac,omitempty"`
+	Driver    string        `json:"driver,omitempty"`
+	Firmware  string        `json:"firmware,omitempty"`
+	BusInfo   string        `json:"bus_info,omitempty"`
+	Model     string        `json:"model,omitempty"`
+	SpeedMbps int           `json:"speed_mbps,omitempty"`
+	MaxMbps   int           `json:"max_mbps,omitempty"`
+	Duplex    string        `json:"duplex,omitempty"`
+	Port      string        `json:"port,omitempty"`
+	MTU       int           `json:"mtu,omitempty"`
+	State     string        `json:"state"`
+	Master    string        `json:"master,omitempty"`
+	Planes    []string      `json:"planes,omitempty"`
+	Neighbor  *LLDPNeighbor `json:"neighbor,omitempty"`
+}
+
+// LLDPNeighbor is what the switch on the other end of a NIC advertises.
+type LLDPNeighbor struct {
+	ChassisID         string   `json:"chassis_id"`
+	SystemName        string   `json:"system_name,omitempty"`
+	SystemDescription string   `json:"system_description,omitempty"`
+	PortID            string   `json:"port_id,omitempty"`
+	PortDescription   string   `json:"port_description,omitempty"`
+	Capabilities      []string `json:"capabilities,omitempty"`
+}
+
+// PlaneFacts is the address a plane uses on this node and the NICs under it.
+type PlaneFacts struct {
+	Plane   string   `json:"plane"`
+	Bridge  string   `json:"bridge,omitempty"`
+	Address string   `json:"address,omitempty"`
+	Prefix  int      `json:"prefix,omitempty"`
+	NICs    []string `json:"nics,omitempty"`
+	MTU     int      `json:"mtu,omitempty"`
+}
+
+// NetTestServeRequest starts a server bound to the node's address on Plane.
+type NetTestServeRequest struct {
+	Plane           string `json:"plane"`
+	DurationSeconds int    `json:"duration_seconds"`
+}
+
+// NetTestServeResponse names the address the server listens on.
+type NetTestServeResponse struct {
+	Address string `json:"address,omitempty"`
+	Port    int    `json:"port,omitempty"`
+	Error   string `json:"error,omitempty"`
+}
+
+// NetTestRunRequest runs the client towards Address from the node's own
+// address on Plane.
+type NetTestRunRequest struct {
+	Plane           string `json:"plane"`
+	Address         string `json:"address"`
+	Port            int    `json:"port"`
+	DurationSeconds int    `json:"duration_seconds"`
+	Streams         int    `json:"streams"`
+}
+
+// NetTestRunResponse is one direction of a pair's measurement.
+type NetTestRunResponse struct {
+	SourceAddress string    `json:"source_address,omitempty"`
+	BitsPerSecond float64   `json:"bits_per_second"`
+	Bytes         int64     `json:"bytes"`
+	Seconds       float64   `json:"seconds"`
+	Streams       int       `json:"streams"`
+	StreamBPS     []float64 `json:"stream_bps,omitempty"`
+	RTTMinMillis  float64   `json:"rtt_min_ms,omitempty"`
+	RTTAvgMillis  float64   `json:"rtt_avg_ms,omitempty"`
+	RTTMaxMillis  float64   `json:"rtt_max_ms,omitempty"`
+	PacketLoss    float64   `json:"packet_loss_percent,omitempty"`
+	LinkMbps      int       `json:"link_mbps,omitempty"`
+	MTU           int       `json:"mtu,omitempty"`
+	Tool          string    `json:"tool,omitempty"`
+	Error         string    `json:"error,omitempty"`
+}

contracts/ecs/v1/messages.go

@@ -126,8 +126,8 @@ type Assign struct {
 	ENIPrivateIP  string `json:"eniPrivateIp,omitempty"`
 	ENISubnetID   string `json:"eniSubnetId,omitempty"`
 	// TaskRoleARN is the task's IAM role (from the taskdef); when set, the agent
-	// serves its credentials at 169.254.170.2 under CredID, defaulting CredID to
-	// the taskID when the scheduler leaves it empty.
+	// serves its credentials at 169.254.170.2 under CredID, a scheduler-minted
+	// secret. An assign without a CredID gets no credentials.
 	CredID      string `json:"credId,omitempty"`
 	TaskRoleARN string `json:"taskRoleArn,omitempty"`
 	// ExecutionRoleARN is the task's execution role (from the taskdef); when set,

contracts/lambda/v1/README.md

@@ -0,0 +1,66 @@
+# Lambda v1 contracts
+
+This package will own the Lambda process contracts declared by ADR-0009:S13.
+It holds the L-GUEST declaration and, in `guest.go`, the connection vocabulary both ends share: the scope, the URI SAN roles, the ALPN identifier and the fw_cfg entry names. The transport and authentication are decided (ADR-0009:S17); the invocation message types and framing are not chosen, so no message types are defined here.
+It is not an AWS tenant-facing API.
+
+## L-GUEST: node executor to guest agent
+
+**Owner:** the Lambda domain.
+
+**Producers and consumers:** the node-local executor (`runtime/compute/lambda`) and the in-guest Lambda agent (`agents/lambda`), in both directions.
+
+**Operations:** bootstrap with scoped material, readiness, invoke, completion, bounded logs, shutdown.
+
+**Transport:** TCP over the microVM's network (ADR-0009:S17).
+The agent dials one connection to a listener the executor owns for the environment, and the guest exposes no listener for host control.
+At most one authenticated control connection is active per environment, and the agent may reconnect; how a prior connection and its in-flight work are handled is open (ADR-0009 M1).
+The network is shared and untrusted: no source address, port or bridge membership is identity or authority.
+Tenant code may be able to reach the listener, but it cannot open the agent's connection or key material, so any connection it makes fails authentication.
+
+**Authentication:** mutual TLS 1.3, bound to the environment ID and realization generation, using material the user runtime cannot read (ADR-0009:S17).
+For each generation the executor creates ephemeral self-signed host and guest certificates whose single URI SAN is `spinifex-lambda://<Region>/<account>/<function ID>/<environment ID>/<generation>/<role>`, with role `host` or `guest`.
+Each end validates the peer certificate against the one pinned peer certificate and separately requires that URI SAN with the opposite role; hostname verification is not relied on.
+ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake.
+Under TLS 1.3 the guest's handshake can complete before the host refuses its certificate, so the guest treats the connection as accepted only after the host's first message.
+
+**Bootstrap material:** the executor passes these fw_cfg entries under `opt/spinifex/lambda/`, and the agent reads them as root and unloads `qemu_fw_cfg` before user code starts.
+
+| Entry | Content |
+|---|---|
+| `guest-key` | Guest private key, PEM |
+| `guest-cert` | Guest certificate, PEM |
+| `host-cert` | The one pinned host certificate, PEM |
+| `expect` | `<Region>/<account>/<function ID>/<environment ID>/<generation>` |
+| `host-addr` | Host listener as `host:port` |
+
+The guest's own interface settings come from `opt/spinifex/netcfg`, the entry other system VMs use.
+The agent unloads `qemu_fw_cfg` even when an entry is missing, and refuses to start if the fw_cfg tree is still readable afterwards.
+Until the message format exists, the agent authenticates and holds the connection but accepts no host message and starts no user code.
+
+**Authority:** one environment's attempt only.
+The user runtime never holds these keys, and sharing the guest with the agent gives user code no platform authority.
+
+**Idempotency and fencing:** an invoke carries the attempt-grant identity and the attempt ID.
+The agent accepts a run command only under a current grant, and a completion under a superseded grant is discarded and reported.
+How a grant is enforced and proven ended is open (ADR-0009 M1).
+
+**Errors and versions:** the agent and executor fail closed on an unknown protocol version or an identity mismatch.
+
+**Readiness:** environment readiness needs authenticated, current-generation agent evidence; a running hypervisor process does not count.
+
+**Payloads:** the contract must carry a synchronous payload and response of up to 6 MB and an asynchronous payload of up to 1 MB, plus framing, over a path whose size bound is declared.
+How payloads move is open (ADR-0009 M3).
+
+### Compatibility boundary
+
+The agent ships inside the runtime image pinned by each published version, so it is outside the coordinated release train.
+The executor supports every L-GUEST major carried by a still-supported runtime image.
+Retiring an image follows a published runtime-deprecation lifecycle that blocks create and update before it blocks invoke.
+
+Evidence required before the first agent image ships: a function on an old image invoked by a new executor, and the reverse during rollback.
+
+## L-RUNTIME
+
+The guest agent serves the AWS Lambda Runtime API, version `2018-06-01`, to the user runtime at `AWS_LAMBDA_RUNTIME_API`.
+Spinifex does not version this AWS contract, and conformance is measured against AWS.

contracts/lambda/v1/guest.go

@@ -0,0 +1,54 @@
+// Package lambdav1 holds the Lambda process-contract vocabulary shared by the
+// node executor and the guest agent. See README.md for the declarations.
+package lambdav1
+
+import (
+	"fmt"
+	"strings"
+)
+
+// ALPNv1 names L-GUEST major version 1; a peer offering any other fails the handshake.
+const ALPNv1 = "spinifex-lambda-guest/1"
+
+// FwCfgPrefix prefixes every fw_cfg entry the executor passes the guest.
+const FwCfgPrefix = "opt/spinifex/lambda/"
+
+// fw_cfg entry names under FwCfgPrefix.
+const (
+	EntryGuestKey  = "guest-key"
+	EntryGuestCert = "guest-cert"
+	EntryHostCert  = "host-cert"
+	EntryScope     = "expect"
+	EntryHostAddr  = "host-addr"
+)
+
+// Roles named as the last URI SAN path segment.
+const (
+	RoleHost  = "host"
+	RoleGuest = "guest"
+)
+
+const uriScheme = "spinifex-lambda"
+
+// Scope names one environment generation:
+// Region/account/function ID/environment ID/generation.
+type Scope string
+
+// ParseScope accepts exactly five non-empty, slash-separated segments.
+func ParseScope(s string) (Scope, error) {
+	parts := strings.Split(s, "/")
+	if len(parts) != 5 {
+		return "", fmt.Errorf("lambdav1: scope %q: want 5 segments, got %d", s, len(parts))
+	}
+	for _, p := range parts {
+		if p == "" || strings.ContainsAny(p, " \t\r\n?#%") {
+			return "", fmt.Errorf("lambdav1: scope %q: empty or invalid segment", s)
+		}
+	}
+	return Scope(s), nil
+}
+
+// URI is the single URI SAN a certificate for role in this scope carries.
+func (s Scope) URI(role string) string {
+	return uriScheme + "://" + string(s) + "/" + role
+}

contracts/lambda/v1/guest_test.go

@@ -0,0 +1,20 @@
+package lambdav1
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseScope(t *testing.T) {
+	s, err := ParseScope("ap-southeast-2/000000000001/fn-1/env-1/gen-1")
+	require.NoError(t, err)
+	assert.Equal(t, "spinifex-lambda://ap-southeast-2/000000000001/fn-1/env-1/gen-1/host", s.URI(RoleHost))
+	for _, bad := range []string{
+		"", "a/b/c/d", "a/b/c/d/e/f", "a//c/d/e", "a/b/c/d/e/", "a/b/c/d/e?x", "a/b/c/d/e#x", "a b/b/c/d/e",
+	} {
+		_, err := ParseScope(bad)
+		assert.Error(t, err, bad)
+	}
+}

contracts/operator/v1/admin.go

@@ -74,3 +74,65 @@ type ListAccountsResponse struct {
 	Accounts []AccountSummary `json:"accounts"`
 	Count    int              `json:"count"`
 }
+
+// IAMQuiescence is the coordinator's recorded view of IAM quiescence for one
+// fence. Recording only: it admits nothing, and QuiescedScope stays empty until
+// a final inventory re-read exists to catch a late write.
+type IAMQuiescence struct {
+	OperationID   string            `json:"operationId"`
+	Fence         uint64            `json:"fence"`
+	Generation    uint64            `json:"generation"`
+	QuiescedScope []string          `json:"quiescedScope"`
+	Unresolved    []UnresolvedScope `json:"unresolved"`
+	Consumers     []ConsumerBasis   `json:"consumers"`
+	Writers       []AccountedWriter `json:"writers"`
+	Inventory     []InventoryStatus `json:"inventory"`
+	EvaluatedAt   string            `json:"evaluatedAt"`
+}
+
+// UnresolvedScope names a scope or writer that blocks quiescence, with a fixed
+// reason. It is never a timeout.
+type UnresolvedScope struct {
+	Scope  string `json:"scope"`
+	Class  string `json:"class"`
+	Reason string `json:"reason"`
+}
+
+// ConsumerBasis cites an enforcement consumer and the accepted contract that
+// lets it count as having applied the generation.
+type ConsumerBasis struct {
+	Consumer string `json:"consumer"`
+	Basis    string `json:"basis"`
+}
+
+// AccountedWriter is a writer that cannot be fenced and is covered by the
+// owner's inventory instead. Ref names the tracked work that closes the gap.
+type AccountedWriter struct {
+	Writer string `json:"writer"`
+	Basis  string `json:"basis"`
+	Ref    string `json:"ref"`
+}
+
+// InventoryStatus is the owner inventory's status for one kind at evaluation.
+type InventoryStatus struct {
+	Kind   string `json:"kind"`
+	Status string `json:"status"`
+}
+
+// Classes of an UnresolvedScope.
+const (
+	UnresolvedClassFence    = "fence"
+	UnresolvedClassConsumer = "consumer"
+	UnresolvedClassWriter   = "writer"
+)
+
+// Fixed reasons of an UnresolvedScope.
+const (
+	ReasonFenceMismatch                = "fence-mismatch"
+	ReasonIAMReadbackFailed            = "iam-readback-failed"
+	ReasonNoAcceptedEnforcement        = "no-accepted-enforcement-contract"
+	ReasonNoCompleteOwnerInventory     = "no-complete-owner-inventory"
+	ReasonParticipantNotDeclared       = "participant-not-declared"
+	ReasonOwnerInventoryIncomplete     = "owner-inventory-incomplete"
+	ReasonFinalInventoryRereadNotWired = "final-inventory-reread-not-wired"
+)

contracts/telemetry/v1/node_live.go

@@ -0,0 +1,53 @@
+package telemetryv1
+
+import "regexp"
+
+// Node live subjects mirror the instance ones, keyed by node name. Only the
+// super-admin account's gateway handler subscribes to them.
+const (
+	nodeLiveWatchSubjectPrefix  = "telemetry.watch.node."
+	nodeLiveSampleSubjectPrefix = "telemetry.live.node."
+)
+
+var liveNodePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
+
+// ValidLiveNode reports whether a node name is safe to place in a subject.
+func ValidLiveNode(node string) bool {
+	return liveNodePattern.MatchString(node)
+}
+
+// NodeLiveWatchSubject is where a viewer asks a node to keep sampling itself.
+// A request on it is answered with a NodeLiveRing.
+func NodeLiveWatchSubject(node string) string {
+	return nodeLiveWatchSubjectPrefix + node
+}
+
+// NodeLiveSampleSubject carries one NodeLiveSample per period while watched.
+func NodeLiveSampleSubject(node string) string {
+	return nodeLiveSampleSubjectPrefix + node
+}
+
+// NodeLiveRing answers a watch with the samples newer than its SinceMillis.
+type NodeLiveRing struct {
+	Samples []NodeLiveSample `json:"samples"`
+}
+
+// NodeLiveSample is one host measurement. Rates are per second over the
+// interval ending at TimeMillis.
+type NodeLiveSample struct {
+	TimeMillis int64   `json:"t"`
+	CPUPercent float64 `json:"cpu"`
+	// MemoryUsedPercent is used memory less page cache, as free(1) reports it.
+	MemoryUsedPercent float64 `json:"mem"`
+	Load1             float64 `json:"load1"`
+	// Planes holds the traffic of each plane's physical NICs, keyed wan, lan, vpc.
+	Planes map[string]NodeLivePlane `json:"planes"`
+}
+
+// NodeLivePlane is one plane's traffic from the host's side.
+type NodeLivePlane struct {
+	NetworkIn  float64 `json:"netIn"`
+	NetworkOut float64 `json:"netOut"`
+	PacketsIn  float64 `json:"pktIn"`
+	PacketsOut float64 `json:"pktOut"`
+}

docs/COMMANDS.md

@@ -152,12 +152,12 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `spx admin account create` | `--name` (required) | Connects to NATS → refuses if an account already holds the name (case- and whitespace-insensitive) → reserves the name in `spinifex-account-names` KV → CAS loop on `spinifex-account-counter:next_id` for sequential 12-digit ID → creates Account record in `spinifex-accounts` KV → creates default VPC → creates `admin` user, AdministratorAccess policy (Action:*, Resource:*) and one access key → prints credentials and writes a `spinifex-<name>` AWS CLI profile |
 | `spx admin account create --remote` | `--name` (required), `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA), `--client-token` (default: generated), `--source` (default: `spx-cli`) | Same account, created through `POST /admin/CreateAccount` over SigV4 instead of NATS — the path the public signup form uses, so a shell can exercise it. Credentials come from the standard AWS chain (env vars or `AWS_PROFILE`), never from the cluster master key; the caller must be an IAM user in the super-admin account holding `spinifex:CreateAccount`. Reuse the printed `--client-token` to retry: a fresh token would create a second account. |
 | `spx admin account list` | — | Connects to NATS → IAMService.ListAccounts() → prints table with Account ID, Name, Status, Created |
-| `spx admin account list --remote` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Same listing read through `POST /admin/ListAccounts` over SigV4 instead of NATS, so an off-cluster operator or the load-test harness sees its tenants without SSH access to a node. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:ListAccounts`. |
+| `spx admin account list --remote` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Same listing read through `POST /admin/ListAccounts` over SigV4 instead of NATS, so an off-cluster operator or the load-test harness sees its tenants without SSH access to a node. Credentials come from the standard AWS credential chain: on a cluster node that means `AWS_PROFILE=spinifex` (there is no default profile, so without it the command fails to find credentials), or equivalent credentials; the caller must be an IAM user in the super-admin account holding `spinifex:ListAccounts`. |
 | `spx admin account quota get <account-id>` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Reads the limits in force for one account through `POST /admin/GetAccountQuota` over SigV4, and reports for each dimension whether it came from the `[quota]` block in `awsgw.toml` (`config`) or from an override stored against the account (`override`). Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:GetAccountQuota`. |
 | `spx admin account quota set <account-id>` | `--vcpus`, `--vpcs`, `--subnets`, `--eips`, `--volumes`, `--volumes-gib`, `--rds-instances`, `--load-balancers`, `--clear`, plus `--endpoint`, `--region`, `--ca-bundle` | Overrides individual limits for one account through `POST /admin/PutAccountQuota`, leaving every dimension not named inheriting the configured baseline. `-1` uncaps a dimension; `0` is a real limit that denies every request on it. `--clear` removes the whole override record and cannot be combined with a limit flag. The account must already exist. Prints the resulting limits, so a caller never has to guess what it applied. The caller must hold `spinifex:PutAccountQuota`. |
-| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record) | Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → seals the account's quota override record so no later write can recreate it, which needs every gateway node the cluster configures to answer a read-only probe first (otherwise nothing is sealed and the account stays `TERMINATING`) → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Platform-managed serving endpoints are reaped ahead of instances, because only their own delete releases the ENI and attached data volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
+| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record), `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Always runs through `POST /admin/DeleteAccount` over SigV4 and follows `DescribeAccountDeletion` to completion; the CLI never tears an account down in its own process, so a CLI delete and an API delete cannot race. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → seals the account's quota override record so no later write can recreate it, which needs every gateway node the cluster configures to answer a read-only probe first (otherwise nothing is sealed and the account stays `TERMINATING`) → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Platform-managed serving endpoints are reaped ahead of instances, because only their own delete releases the ENI and attached data volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
 | `spx admin account delete --force` | as above | Escalates only for resources the ordinary API refuses: it clears a volume attachment in the control plane without the guest's cooperation (`ec2.ForceDetachVolume`), hard-destroys after a graceful attempt times out, and treats an already-missing resource as deleted. It never reorders the stages and never widens scope beyond the account. Use it once the guest is being destroyed — this is the escape from a volume stuck attached to an instance that will not terminate, where neither can otherwise be deleted. |
-| `spx admin account delete --remote` | as above, plus `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Same teardown driven through `POST /admin/DeleteAccount` over SigV4 instead of NATS, then follows `DescribeAccountDeletion` to completion, printing each stage as it lands. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. |
+| `spx admin account delete --remote` | as above | Accepted for existing scripts and ignored: every delete already goes through the admin API. |
 | `spx admin principal create <name>` | `--grant` (repeatable or comma-separated; default: every admin method — `CreateAccount`, `DeleteAccount`, `DescribeAccountDeletion`, `GetAccountQuota`, `ListAccounts`, `PutAccountQuota`) | Creates an IAM user in the super-admin account with an inline `spx-admin-methods` policy allowing the requested methods, revokes any access key left from a previous run, and mints one new key. Each method is named rather than granted as `spinifex:*`, so a later addition to the admin surface is not authorised by an existing key. The principal's grants are exactly that one policy: re-running with fewer `--grant` values narrows it, and any other inline policy on the user is removed, so a rename cannot leave a wider grant behind. The secret is printed once; store it in a secret manager or an AWS profile, never in a config file. `admin` is refused as a name — it is the account's own administrator and rotating its key would revoke the bootstrap credential. |
 | `spx admin principal list` | — | Lists every principal in the super-admin account with its access-key count and what it may call. An attached policy is reported by name (`AdministratorAccess (attached)`) rather than expanded, which is how an unscoped principal is spotted. |
 | `spx admin principal revoke <name>` | — | Deletes every access key the named principal holds, leaving the user and its policy in place. Immediate and cluster-wide; re-issue with `principal create`. |

docs/admin/account-management/README.md

@@ -236,12 +236,12 @@ Each stage runs to empty before the next begins, so nothing is deleted while som
 A large account takes minutes to tear down. The API does not hold the connection open for it: `DeleteAccount` starts the job, returns a `deletionId` immediately, and progress is read from `DescribeAccountDeletion`.
 
 ```bash
-AWS_PROFILE=spinifex-operator spx admin account delete 000000000042 --remote \
+AWS_PROFILE=spinifex-operator spx admin account delete 000000000042 \
   --name customer@example.com --yes \
   --endpoint https://node1.example.com:9999 --region us-west-1
 ```
 
-`spx --remote` starts the job and then polls, printing each stage as it lands. Your own client can do the same, or start the job and check back later — the deletion record outlives the account, so `DescribeAccountDeletion` stays answerable after the account is gone.
+`spx admin account delete` always goes through the admin API, so it signs with the standard AWS credential chain. On a cluster node that means `AWS_PROFILE=spinifex` (there is no default profile, so without it the command cannot find credentials), or equivalent credentials for an IAM user holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`, and a reachable gateway. The older `--remote` flag is accepted and ignored. The command starts the job and then polls, printing each stage as it lands. Your own client can do the same, or start the job and check back later — the deletion record outlives the account, so `DescribeAccountDeletion` stays answerable after the account is gone.
 
 One job runs per account. A second `DeleteAccount` with a different token while one is running returns `OperationInProgress`; with the *same* token it returns the running job, which is what makes a retry safe.
 

docs/containers/eks/README.md

@@ -58,7 +58,7 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM
 **Spinifex specifics**
 
 - **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
-- **Security groups are auto-managed.** Spinifex creates the cluster and node-group security groups deterministically (e.g. `eks-cluster-<name>-nodegroup-sg`); `vpc_config.security_group_ids` is ignored. To expose a workload you add an ingress rule to the auto-managed node-group SG.
+- **The cluster security group is auto-managed.** As on EKS, Spinifex creates one `eks-cluster-sg-<name>-<id>` in your VPC, tagged `kubernetes.io/cluster/<name>=owned` and `aws:eks:cluster-name`, allowing all traffic from itself. It is attached to the workers and the private-endpoint network interface, and `DescribeCluster` reports it as `clusterSecurityGroupId`. A launch template that names security groups replaces it on the workers, as on AWS. Clusters created before this change keep their `-control-plane-sg` and `-nodegroup-sg` until you recreate them. Deleting a cluster deletes the cluster security group and removes only the rules on your other security groups that reference it. `vpc_config.security_group_ids` is ignored.
 - **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
 - **Workers launch with IMDSv2 required and an IMDS hop limit of 2.**
   This matches AWS managed node groups and lets a pod without IRSA reach the node role through the SDK default credential chain.
@@ -143,6 +143,25 @@ No user-facing operation replaces a control plane, and automatic replacement doe
 Recreate the cluster to adopt it.
 User-workload injection through a cluster created with an older image is not measured.
 
+### Add-on versions
+
+Add-on versions use the AWS form `v<x.y.z>-eksbuild.<n>`, for example `v1.40.1-eksbuild.1`.
+`DescribeAddonVersions` returns versions in this form, newest first, and every other add-on response reports the version the same way.
+Spinifex ships one EKS build of each upstream version, so `<n>` is `1` today.
+
+`DescribeAddonVersions` honours `addonName`, `kubernetesVersion`, `types`, `publishers`, `owners`, `maxResults` and `nextToken`.
+Each version carries `compatibilities` with the `clusterVersion`, the `platformVersions`, and a `defaultVersion` flag that is true for exactly one version per add-on and cluster version.
+A `kubernetesVersion` that Spinifex does not run returns no add-ons.
+
+`CreateAddon` and `UpdateAddon` accept the AWS form of a version in the catalogue.
+They also accept the bare upstream form, such as `1.40.1`, and Spinifex logs each use at Info.
+This is a Spinifex leniency that AWS does not offer, and it exists so that older scripts keep working.
+Any other value is refused with a message that names `addonVersion` and lists the supported versions, for example `v1.40.1-eksbuild.2`, `1.40` or `v1.40.1`.
+
+In Terraform, pass the AWS form to `aws_eks_addon.addon_version`, either as a literal or from `data.aws_eks_addon_version`.
+The AWS provider validates the format before it calls the API, so it rejects the bare form.
+Leaving `addon_version` unset still installs the catalogue default.
+
 ## Prerequisites
 
 > [!IMPORTANT]
@@ -225,12 +244,11 @@ The list shows the node group workers only. The control-plane VMs never appear a
 
 ### 4. Expose a workload
 
-Workers join an auto-managed security group that only allows intra-cluster traffic. To reach a NodePort service, add one ingress rule to that SG (look it up by its deterministic name):
+Workers join the cluster security group, which only allows intra-cluster traffic. To reach a NodePort service, add one ingress rule to it (`DescribeCluster` reports its id):
 
 ```bash
-SG=$(aws ec2 describe-security-groups \
-  --filters Name=group-name,Values=eks-cluster-demo-nodegroup-sg \
-  --query 'SecurityGroups[0].GroupId' --output text)
+SG=$(aws eks describe-cluster --name demo \
+  --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text)
 
 aws ec2 authorize-security-group-ingress \
   --group-id "$SG" --protocol tcp --port 30080 --cidr 0.0.0.0/0
@@ -272,7 +290,7 @@ The form notes that the authentication mode is API and that the legacy aws-auth
 
 - The **Access** tab lists IAM principals and their cluster-access policies — add an access entry to let another user or role reach the cluster.
 - The **Addons** tab installs and shows the health of managed addons.
-- The **Networking** tab shows the cluster's VPC, subnets, endpoint access, and the auto-managed security groups.
+- The **Networking** tab shows the cluster's VPC, subnets, endpoint access, and the auto-managed cluster security group.
 
 To run `kubectl` against the cluster, follow the AWS CLI tab's `update-kubeconfig` step.
 
@@ -355,11 +373,11 @@ Control plane readiness and kubectl authentication above lists which clusters re
 - Confirm the workers can pull images — a default route to an Internet Gateway (public) or NAT Gateway (private).
 - Confirm the node role has `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, and `AmazonEC2ContainerRegistryReadOnly`.
 
-**Workload unreachable from outside the cluster.** The node-group security group only allows intra-cluster traffic by default. Add an ingress rule for your NodePort to `eks-cluster-<name>-nodegroup-sg`, as shown in the CLI tab.
+**Workload unreachable from outside the cluster.** The cluster security group only allows intra-cluster traffic by default. Add an ingress rule for your NodePort to it, as shown in the CLI tab. An Ingress served by the AWS Load Balancer Controller needs no rule: the controller adds it to the cluster security group.
 
 **`create-cluster` rejected with an authentication-mode error.** `CONFIG_MAP` is the one mode Spinifex rejects, because it has no `aws-auth` ConfigMap path. Set `authentication_mode` to `API` or `API_AND_CONFIG_MAP`, or leave it unset, and grant access with access entries.
 
-**Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, leave `addon_version` unset so Spinifex selects the catalog default.
+**Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, use the AWS form from `DescribeAddonVersions` (see Add-on versions above), or leave `addon_version` unset so Spinifex selects the catalog default.
 
 ## Reference Architectures
 

docs/coverage/README.md

@@ -17,7 +17,7 @@ tags:
 
 ## Overview
 
-The platform serves **405 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
+The platform serves **406 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
 
 | Service | Operations |
 |---|---:|
@@ -27,9 +27,9 @@ The platform serves **405 operations** across the AWS APIs below. Every page nam
 | [ECS](/coverage/ecs) | 31 |
 | [EKS](/coverage/eks) | 27 |
 | [ELBv2](/coverage/elbv2) | 36 |
-| [IAM](/coverage/iam) | 81 |
+| [IAM](/coverage/iam) | 82 |
 | [RDS](/coverage/rds) | 27 |
 | [Route 53](/coverage/route53) | 13 |
 | [S3](/coverage/s3) | 26 |
 | [STS](/coverage/sts) | 5 |
-| **Total** | **405** |
+| **Total** | **406** |

docs/coverage/coverage.json

@@ -2938,7 +2938,8 @@
           "UpdateAccessKey",
           "UpdateAssumeRolePolicy",
           "UpdateRole",
-          "UpdateRoleDescription"
+          "UpdateRoleDescription",
+          "UpdateUser"
         ],
         "implemented": [
           "AddRoleToInstanceProfile",
@@ -3021,7 +3022,8 @@
           "UpdateAccessKey",
           "UpdateAssumeRolePolicy",
           "UpdateRole",
-          "UpdateRoleDescription"
+          "UpdateRoleDescription",
+          "UpdateUser"
         ],
         "stubbed": [],
         "unsupported": [],
@@ -3121,7 +3123,6 @@
           "UpdateServerCertificate",
           "UpdateServiceSpecificCredential",
           "UpdateSigningCertificate",
-          "UpdateUser",
           "UploadSSHPublicKey",
           "UploadServerCertificate",
           "UploadSigningCertificate"

docs/coverage/iam/README.md

@@ -18,7 +18,7 @@ tags:
 
 ## Overview
 
-Spinifex implements **81 operations** in the IAM `2010-05-08` API model.
+Spinifex implements **82 operations** in the IAM `2010-05-08` API model.
 
 ### Scope
 
@@ -109,3 +109,4 @@ All IAM operations are account-scoped. The root user of an account bypasses poli
 | `UpdateAssumeRolePolicy` |
 | `UpdateRole` |
 | `UpdateRoleDescription` |
+| `UpdateUser` |

docs/coverage/internal-report.md

@@ -987,7 +987,7 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 
 ## iam
 
-`2010-05-08` — 81 of 180 modelled operations implemented (45.0%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
+`2010-05-08` — 82 of 180 modelled operations implemented (45.6%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
 
 ### Not implemented
 
@@ -1069,7 +1069,6 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 - `UpdateOpenIDConnectProviderThumbprint`
 - `UpdateSAMLProvider`
 - `UpdateServerCertificate`
-- `UpdateUser`
 - `UploadServerCertificate`
 
 ### Not applicable — do not build

docs/package-boundary/eks-addon-lifecycle.md

@@ -20,7 +20,7 @@ There is no generation, so the record mixes desired fields with the latest obser
 | Durable record | `clusters/{cluster}/addons/{addon}` holds `Record` (`domains/eks/addon/record.go`): `addonName`, `addonVersion`, `status`, `serviceAccountRoleArn`, `configurationValues`, `health`, `arn`, `tags`, `createdAt`, `modifiedAt` (the optional ones `omitempty`). `clusters/{cluster}/addons/{addon}/manifest` holds `Manifest` (`domains/eks/addon/record.go`): `addonName`, `addonVersion`, `serviceAccountRoleArn`, `configurationValues`. Listing skips the manifest sub-key by segment count. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestAddonRecord_DecodesHandWrittenJSON`, `domains/eks/addon/store_test.go` `TestRecord_RoundTrip`, `TestList_SkipsManifestSubKeys` |
 | Writers | Record: the owner in `domains/eks/addon/owner.go`, reached from the `CreateAddon`, `UpdateAddon` and `DeleteAddon` handlers, the cluster reconciler's report relay and the GPU node group launch: `Owner.Create` (unconditional put), `Owner.Update` and its failure marking (revision CAS), `Owner.Delete` (get then delete), `Owner.ApplyReport` (revision CAS), `Owner.EnsureGPUDevicePlugin` through `Owner.Create`; and the cluster prefix purge (`DeleteClusterChildren`). Manifest: `StagingInstaller.Install` (unconditional put on create and update), `StagingInstaller.Uninstall` (`domains/eks/addon/delivery.go`), `Owner.Delete` and the prefix purge. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestUpdateAddon_ReportsSuccessfulWhileRecordIsUpdating`, `TestDeleteAddon_ReturnsDeletingAfterErasingRecordAndManifest`, `TestDeleteClusterChildren_SweepsAddonRecordsAndManifestsScopedToCluster`, `domains/eks/addon/store_test.go` `TestCasUpdate`, `TestDeleteRecord_RemovesRecordAndManifest` |
 | Readers | `ListAddons`, `DescribeAddon`, `CreateAddon` (existence check) and the reconciler read the record. `ListStagedAddonManifests` reads every `/manifest` key for the cluster (`addon.ListManifests`); it is served on `eks.ListStagedAddonManifests` (`daemon/daemon.go`) and reached only through the gateway's internal-addons route. | `handlers/eks/addons_test.go` `TestAddon_DescribeListDelete`, `TestListStagedAddonManifests` |
-| Catalogue | Static in-binary catalogue (`domains/eks/addon/catalog.go`): `aws-load-balancer-controller`, `argocd`, `aws-ebs-csi-driver`, and hidden-but-creatable `nvidia-device-plugin` and `spinifex-noop`; one version each, the first being the default. Create and update reject unknown names and versions with `InvalidParameterValue`. | `domains/eks/addon/catalog_test.go`, `handlers/eks/addons_test.go` `TestDescribeAddonVersions_ReturnsCatalog`, `TestCreateAddon_UnknownAddonRejected`, `TestCreateAddon_UnknownVersionRejected`, `TestUpdateAddon_UnknownVersionRejected` |
+| Catalogue | Static in-binary catalogue (`domains/eks/addon/catalog.go`): `aws-load-balancer-controller`, `argocd`, `aws-ebs-csi-driver`, and hidden-but-creatable `nvidia-device-plugin` and `spinifex-noop`; one version each, the first being the default. Versions are stored and used internally in the bare upstream form, and API responses show the AWS form `v<x.y.z>-eksbuild.<n>` (`Spec.AWSVersion`). Create and update accept the AWS form or the exact bare form and reject unknown names and other version strings with `InvalidParameterValue`, which the gateway returns as `InvalidParameterException`. | `domains/eks/addon/catalog_test.go`, `handlers/eks/addons_test.go` `TestDescribeAddonVersions_ReturnsCatalog`, `TestCreateAddon_UnknownAddonRejected`, `TestCreateAddon_UnknownVersionRejected`, `TestUpdateAddon_UnknownVersionRejected` |
 | Desired state | Version, `serviceAccountRoleArn`, `configurationValues` and create-time tags. `resolveConflicts`, `podIdentityAssociations` and `clientRequestToken` are accepted and dropped; `preserve` on delete is ignored. `configurationValues` reaches the guest but the agent discards it (`mulga-eks-addon-sync.sh` `sync_once`), so it has no effect. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestDeleteAddon_ReturnsDeletingAfterErasingRecordAndManifest` |
 | Observed state | `status` and `health` are overwritten by guest reports on the same record as the desired fields. `health` is the latest failure message, projected as one `AddonIssue` with only `message`. Statuses held: `CREATING`, `ACTIVE`, `UPDATING`, `DEGRADED`, `CREATE_FAILED`; `DELETING` is defined but never stored. | `TestAddonRecord_DecodesHandWrittenJSON`, `handlers/eks/addon_status_reconciler_test.go` `TestApplyAddonStatusReport` |
 | Generation | None. Reports carry `version` and `ts`, and neither is read: any report for the add-on name applies to whatever is currently desired. | `TestAddonStatusReport_MismatchedVersionIsAccepted` |

docs/security/network-connections/README.md

@@ -99,6 +99,7 @@ behavior in other words.
 | 4432 | Formation server | HTTPS | Cluster (bootstrap only) | Cluster join coordination; active only while a join token is valid. Binds the node's `--bind` (lan) address. See *Formation port lifecycle* below. | Short-lived bearer token + TLS¹ |
 | 4222 | spinifex-nats (client) | NATS + TLS | Cluster | Internal service bus for EC2/EBS/VPC/S3 handlers | Token + mutual TLS (cluster CA) |
 | 4248 | spinifex-nats (cluster) | NATS + TLS | Cluster | Inter-node NATS federation | Token + mutual TLS (cluster CA) |
+| 5001 | iperf2 (admin network test) | TCP | Cluster, any plane | Throughput test between two nodes, started only by the super-admin account. Listens on one plane address for the length of one test, then exits. Runs with no capabilities. | None — peer-scoped by firewall, and the client may only target its own plane subnet |
 | 5300 | northstar | DNS (UDP + TCP) | Cluster | Forward target for every node's per-instance DNS shim, dialled cross-node. Binds the wildcard by design. | None — restrict by firewall |
 | 6660 | predastore (blob node) | QUIC / UDP | Cluster | Erasure-coded object shard transport between hosts. Multi-node clusters only — see *Predastore ports* below. | TLS 1.3, server certificate verified against the cluster CA |
 | 7660 | predastore (meta node) | QUIC / UDP | Cluster | Raft consensus over global state — buckets and the object index — between hosts. Multi-node clusters only. | TLS 1.3, server certificate verified against the cluster CA |
@@ -221,7 +222,7 @@ table inet spinifex_filter {
 
     # Cluster, from peer nodes only. Replace with your nodes' lan-plane
     # addresses — a CIDR does not generalise to nodes on different subnets.
-    ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
+    ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } tcp dport { 4222, 4248, 4432, 5001, 5300, 6641, 6642, 6643, 6644, 8660 } accept
     ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } udp dport { 5300, 6660, 7660 } accept
 
     # Encap, from peer chassis only. Replace with your nodes' vpc-plane

docs/terraform-workbooks/eks-gitops-argocd/README.md

@@ -79,7 +79,8 @@ Terraform manages the cluster, the addons, the ACM cert, and the HTTPS Ingress;
 - The `argocd` and `aws-ebs-csi-driver` bundles must be baked into the `eks-node` AMI — `describe-addon` returns `no baked bundle` otherwise.
 - The EBS-CSI default StorageClass uses `provisioner: ebs.csi.aws.com`; a PVC against it provisions a **Viperblock-backed EBS volume**.
 - If the PVC stays `Pending`, the app still serves (the counter falls back to in-memory) — see [Troubleshooting](#troubleshooting).
-- Leave addon versions unset (the AWS provider and the catalog disagree on the version-string format).
+- Addon versions are unset, so Spinifex installs each catalog default.
+  To pin one, use the AWS form that `DescribeAddonVersions` returns, such as `v3.0.23-eksbuild.1`.
 - The Argo CD `Application` is a `kubernetes_manifest`, so the `argoproj.io` CRDs must exist before `apply` — apply the parent and let the addon reach `ACTIVE` first.
 
 **Prerequisites:**

docs/terraform-workbooks/eks-gitops-argocd/main.tf

@@ -471,8 +471,7 @@ resource "aws_eks_cluster" "this" {
   # expects: access_config reads as absent (and its ForceNew
   # bootstrap_cluster_creator_admin_permissions would replace the live cluster on
   # re-add), and vpc_config.security_group_ids reads back as drift. Ignore both so
-  # later applies (e.g. adding the Argo CD NodePort rule) neither replace the
-  # cluster nor detach its security groups.
+  # later applies neither replace the cluster nor detach its security groups.
   lifecycle {
     ignore_changes = [access_config, vpc_config[0].security_group_ids]
   }
@@ -520,8 +519,8 @@ resource "aws_eks_node_group" "workers" {
 #
 # Spinifex wires the controller's AWS credentials and ELB-eligible subnets at the
 # node level, so no IRSA role or subnet tagging is needed here. addon_version is
-# omitted: the AWS provider demands a v-prefixed version that the catalog rejects,
-# so let Spinifex default to its catalog version.
+# omitted so Spinifex installs its catalog default. To pin one, use the AWS form
+# from DescribeAddonVersions, for example v2.11.0-eksbuild.1.
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "lbc" {
@@ -542,7 +541,7 @@ resource "aws_eks_addon" "lbc" {
 # Installed through the EKS API; Spinifex stages the bundle host-side and the
 # worker renders it into the K3s auto-deploy dir. The workloads module registers
 # the git repo and creates the Argo CD Application that syncs the demo app.
-# addon_version omitted (see the LBC note).
+# addon_version omitted: the catalog default is installed (see the LBC note).
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "argocd" {
@@ -562,7 +561,7 @@ resource "aws_eks_addon" "argocd" {
 #
 # Ships a default gp3 StorageClass (provisioner ebs.csi.aws.com). A PVC in the
 # demo app's git manifests dynamically provisions a Viperblock-backed EBS volume
-# so the app's state survives pod restarts. addon_version omitted (see LBC note).
+# so the app's state survives pod restarts. addon_version omitted: catalog default.
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "ebs_csi" {
@@ -642,53 +641,6 @@ resource "aws_acm_certificate" "ingress" {
   }
 }
 
-# ---------------------------------------------------------------------------
-# Let the ALB reach the workers' NodePort
-#
-# Spinifex auto-manages the nodegroup SG and admits only intra-cluster traffic.
-# The LBC-provisioned ALB lives in this VPC, so admit the VPC CIDR on the
-# NodePort. Look the SG up by its deterministic name and add one rule.
-# ---------------------------------------------------------------------------
-
-data "aws_security_group" "nodegroup" {
-  filter {
-    name   = "group-name"
-    values = ["eks-cluster-${var.cluster_name}-nodegroup-sg"]
-  }
-
-  filter {
-    name   = "vpc-id"
-    values = [aws_vpc.main.id]
-  }
-
-  depends_on = [aws_eks_node_group.workers]
-}
-
-resource "aws_vpc_security_group_ingress_rule" "nodeport_from_vpc" {
-  security_group_id = data.aws_security_group.nodegroup.id
-  cidr_ipv4         = aws_vpc.main.cidr_block
-  from_port         = var.node_port
-  to_port           = var.node_port
-  ip_protocol       = "tcp"
-
-  tags = {
-    Name = "${var.cluster_name}-alb-nodeport"
-  }
-}
-
-# Same, for the Argo CD UI NodePort its ALB forwards to.
-resource "aws_vpc_security_group_ingress_rule" "argocd_nodeport_from_vpc" {
-  security_group_id = data.aws_security_group.nodegroup.id
-  cidr_ipv4         = aws_vpc.main.cidr_block
-  from_port         = var.argocd_node_port
-  to_port           = var.argocd_node_port
-  ip_protocol       = "tcp"
-
-  tags = {
-    Name = "${var.cluster_name}-argocd-nodeport"
-  }
-}
-
 # ---------------------------------------------------------------------------
 # Outputs
 # ---------------------------------------------------------------------------

docs/terraform-workbooks/eks-https-ingress/README.md

@@ -64,7 +64,6 @@ The LBC is installed as a managed **addon** (`aws_eks_addon`). Spinifex wires th
 | Node Group | `workers` | `node_desired_size` `t3.large` worker(s) — 1 or 3 |
 | Addon | `aws-load-balancer-controller` | Provisions the ALB from the Ingress |
 | ACM Certificate | `eks-https-ingress` | Self-signed, imported; attached to the HTTPS listener |
-| SG Ingress Rule | `eks-https-alb-nodeport` | Admits the VPC CIDR to the workers' NodePort |
 | K8s Deployment + Service | `spinifex-demo` | Themed demo image, 2 replicas, NodePort |
 | K8s Ingress | `spinifex-demo` | `ingressClassName: alb`, HTTPS via the ACM cert |
 
@@ -72,9 +71,10 @@ The LBC is installed as a managed **addon** (`aws_eks_addon`). Spinifex wires th
 
 - **The LBC needs no IRSA or subnet tagging here.** Credentials and ELB-eligible subnets are injected by Spinifex at the node level and into the `alb` `IngressClassParams`. The Ingress just sets `ingressClassName: alb` plus the `alb.ingress.kubernetes.io/*` annotations.
 - **`spinifex.io/managed-ingress = "false"`** disables the K3s built-in traefik/servicelb so the LBC owns ingress, matching AWS parity. Omit the tag (or set it `true`) and the built-in ingress stays on.
-- The ALB targets the workers' **NodePort** (`target-type: instance`). The one SG rule admits the VPC CIDR on that port so the in-VPC ALB can reach the workers; the worker SG is otherwise auto-managed.
+- The ALB targets the workers' **NodePort** (`target-type: instance`). The AWS Load Balancer Controller adds the ALB-to-NodePort rule to the cluster security group itself, so the workbook creates no security group rule.
 - Spinifex ACM supports **ImportCertificate only** — the cert is generated by the `tls` provider and imported, not requested.
-- **Leave `addon_version` unset** (see the [quickstart/addons notes](/docs/eks-quickstart)) — the AWS provider and the Spinifex catalog disagree on the version string format, so let the server choose.
+- **`addon_version` is unset**, so Spinifex installs its catalog default.
+  To pin a version, use the AWS form that `DescribeAddonVersions` returns, such as `v2.11.0-eksbuild.1`, or read it from `data.aws_eks_addon_version`.
 
 **Prerequisites:**
 
@@ -179,16 +179,17 @@ Confirm the addon is `ACTIVE` and the cluster carries `spinifex.io/managed-ingre
 
 ### HTTPS Loads but Returns 502 / 504
 
-The ALB reached the workers but the backend didn't answer. Check the demo pods and the NodePort SG rule:
+The ALB reached the workers but the backend didn't answer. Check the demo pods and the rule the controller added to the cluster security group:
 
 ```bash
 kubectl get pods -o wide
 kubectl get svc spinifex-demo
-aws ec2 describe-security-groups --filters "Name=group-name,Values=eks-cluster-eks-https-nodegroup-sg" \
+SG=$(aws eks describe-cluster --name eks-https --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text)
+aws ec2 describe-security-groups --group-ids "$SG" \
   --query 'SecurityGroups[0].IpPermissions'
 ```
 
-The `eks-https-alb-nodeport` rule must admit the VPC CIDR on the NodePort, and the target group health check (`/healthz`) must pass.
+The cluster security group must admit the load balancer's security group on the NodePort, and the target group health check (`/healthz`) must pass.
 
 ### Addon Fails With "no baked bundle"
 

docs/terraform-workbooks/eks-https-ingress/main.tf

@@ -16,7 +16,8 @@
 # The workloads/ module then creates a Deployment, a NodePort Service, and an
 # Ingress (ingressClassName: alb). The LBC reconciles that Ingress into an
 # internet-facing ALB that terminates TLS with the ACM cert and forwards to the
-# workers' NodePort. The cluster's ELB-eligible subnets are injected by Spinifex
+# workers' NodePort, adding the ALB-to-NodePort rule to the cluster security group
+# itself. The cluster's ELB-eligible subnets are injected by Spinifex
 # into the alb IngressClassParams, so the Ingress needs no subnet annotations.
 #
 # Usage:
@@ -446,8 +447,8 @@ resource "aws_eks_node_group" "workers" {
 #
 # Spinifex wires the controller's AWS credentials and ELB-eligible subnets at the
 # node level, so no IRSA role or subnet tagging is needed here. addon_version is
-# omitted: the AWS provider demands a v-prefixed version that the catalog rejects,
-# so let Spinifex default to its catalog version.
+# omitted so Spinifex installs its catalog default. To pin one, use the AWS form
+# from DescribeAddonVersions, for example v2.11.0-eksbuild.1.
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "lbc" {
@@ -503,40 +504,6 @@ resource "aws_acm_certificate" "ingress" {
   }
 }
 
-# ---------------------------------------------------------------------------
-# Let the ALB reach the workers' NodePort
-#
-# Spinifex auto-manages the nodegroup SG and admits only intra-cluster traffic.
-# The LBC-provisioned ALB lives in this VPC, so admit the VPC CIDR on the
-# NodePort. Look the SG up by its deterministic name and add one rule.
-# ---------------------------------------------------------------------------
-
-data "aws_security_group" "nodegroup" {
-  filter {
-    name   = "group-name"
-    values = ["eks-cluster-${var.cluster_name}-nodegroup-sg"]
-  }
-
-  filter {
-    name   = "vpc-id"
-    values = [aws_vpc.main.id]
-  }
-
-  depends_on = [aws_eks_node_group.workers]
-}
-
-resource "aws_vpc_security_group_ingress_rule" "nodeport_from_vpc" {
-  security_group_id = data.aws_security_group.nodegroup.id
-  cidr_ipv4         = aws_vpc.main.cidr_block
-  from_port         = var.node_port
-  to_port           = var.node_port
-  ip_protocol       = "tcp"
-
-  tags = {
-    Name = "${var.cluster_name}-alb-nodeport"
-  }
-}
-
 # ---------------------------------------------------------------------------
 # Outputs
 # ---------------------------------------------------------------------------

docs/terraform-workbooks/eks-quickstart/README.md

@@ -45,7 +45,7 @@ Under the hood it keeps things minimal: a VPC with two public subnets, an IAM ro
 - Creating the EKS cluster and worker IAM roles with faithful managed-policy attachments
 - Provisioning an `aws_eks_cluster` and a managed `aws_eks_node_group`
 - Pointing the Kubernetes provider at the new cluster and deploying a workload from the same `apply`
-- Opening one rule on the auto-managed worker security group to expose a NodePort
+- Opening one rule on the cluster security group to expose a NodePort
 
 **What gets created**
 
@@ -58,13 +58,13 @@ Under the hood it keeps things minimal: a VPC with two public subnets, an IAM ro
 | ECR Repository | `spinifex-demo` | Holds the demo image the workers pull |
 | EKS Cluster | `eks-quickstart` | Public API endpoint, API auth mode, Kubernetes 1.32 |
 | Node Group | `default` | `node_desired_size` `t3.medium` worker(s) — 1 or 3 |
-| SG Ingress Rule | `eks-quickstart-demo-nodeport` | Opens the demo NodePort on the auto-managed worker SG |
+| SG Ingress Rule | `eks-quickstart-demo-nodeport` | Opens the demo NodePort on the cluster security group |
 | K8s Deployment | `spinifex-demo` | The themed demo image from ECR, 2 replicas |
 | K8s Service | `spinifex-demo` | NodePort publishing the demo on the worker |
 
 **Spinifex specifics**
 
-- The cluster's security groups are **auto-managed** — `vpc_config.security_group_ids` is ignored. To reach a NodePort, this workbook looks the worker SG up by its deterministic name (`eks-cluster-<name>-nodegroup-sg`) and adds a single ingress rule.
+- The cluster security group is **auto-managed**, as on EKS: Spinifex creates `eks-cluster-sg-<name>-<id>`, attaches it to the workers, and reports it as `vpc_config[0].cluster_security_group_id`. `vpc_config.security_group_ids` is ignored. To reach a NodePort, this workbook adds a single ingress rule to that security group.
 - The worker AMI is always Spinifex's `eks-node` image; `ami_type` is recorded but does **not** select the image.
 - `authentication_mode` can be `"API"` or `"API_AND_CONFIG_MAP"`, and bare `"CONFIG_MAP"` is rejected. Access is granted through access entries, as there is no `aws-auth` ConfigMap path.
 - The workers need outbound internet (here via the IGW) to pull the demo container image.
@@ -182,8 +182,9 @@ aws ec2 describe-instances --filters "Name=tag:spinifex:eks-cluster,Values=eks-q
 # Did the demo pods roll out?
 kubectl get pods -o wide
 
-# Is the NodePort rule on the worker SG?
-aws ec2 describe-security-groups --filters "Name=group-name,Values=eks-cluster-eks-quickstart-nodegroup-sg" \
+# Is the NodePort rule on the cluster security group?
+SG=$(tofu output -raw cluster_security_group_id)
+aws ec2 describe-security-groups --group-ids "$SG" \
   --query 'SecurityGroups[0].IpPermissions'
 ```
 

docs/terraform-workbooks/eks-quickstart/main.tf

@@ -310,29 +310,15 @@ resource "aws_eks_node_group" "default" {
 }
 
 # ---------------------------------------------------------------------------
-# Open the NodePort on the auto-managed nodegroup SG
+# Open the NodePort on the cluster security group
 #
-# Spinifex creates the worker SG itself (vpc_config.security_group_ids is
-# ignored) and admits only intra-cluster traffic. To reach the demo NodePort
-# from a browser, look the SG up by its deterministic name and add one rule.
+# Spinifex creates the cluster security group itself, as EKS does, and attaches
+# it to the workers. It admits only intra-cluster traffic. To reach the demo
+# NodePort from a browser, add one rule to the SG the cluster reports.
 # ---------------------------------------------------------------------------
 
-data "aws_security_group" "nodegroup" {
-  filter {
-    name   = "group-name"
-    values = ["eks-cluster-${var.cluster_name}-nodegroup-sg"]
-  }
-
-  filter {
-    name   = "vpc-id"
-    values = [aws_vpc.main.id]
-  }
-
-  depends_on = [aws_eks_node_group.default]
-}
-
 resource "aws_vpc_security_group_ingress_rule" "nodeport" {
-  security_group_id = data.aws_security_group.nodegroup.id
+  security_group_id = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id
   cidr_ipv4         = var.browse_cidr
   from_port         = var.node_port
   to_port           = var.node_port
@@ -367,6 +353,10 @@ output "region" {
   value = var.region
 }
 
+output "cluster_security_group_id" {
+  value = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id
+}
+
 output "node_port" {
   value = var.node_port
 }

docs/ui/design-system.md

@@ -24,7 +24,7 @@ All colours are OKLCH and come from mulgadc.com, so product and site match.
 | Token | Use |
 |---|---|
 | `background` | The app page. Warm off-white in light, ink `#14181D` in dark. |
-| `card`, `popover` | Raised surfaces. White in light, about `#1B2027` in dark. |
+| `card`, `popover` | Raised surfaces. White in light, about `#1B2027` in dark. `popover` is every floating surface: tooltips, popovers, menus and pickers. |
 | `muted`, `secondary`, `accent` | Hover rows, secondary buttons, quiet fills. |
 | `border` | Every divider and card edge. `#DCD8CE` in light, `oklch(1 0 0 / 8%)` in dark. |
 | `brand` | Brand orange `#EE4B12`, for the logo and decoration only. It fails text contrast, so it never carries text. |
@@ -39,15 +39,22 @@ Each status has a solid token plus `-foreground` (text on a tinted background),
 
 | Status | Hue | AWS states |
 |---|---|---|
-| `success` | green | running, available, completed, healthy |
-| `warning` | brand yellow `#F5B500` | pending, stopping, creating, modifying, deleting, draining |
-| `danger` | cool red, hue 17 | stopped, error, failed, unhealthy, unavailable |
+| `success` | green | settled and good: running, available, active, associated, attached, in-use, healthy, ok, main |
+| `warning` | brand yellow `#F5B500` | changing state and will settle: pending, stopping, creating, modifying, deleting, draining |
+| `caution` | orange, hue 58 | works but needs attention: degraded, impaired, insufficient-data |
+| `danger` | cool red, hue 17 | stopped, error, failed, update-failed, unhealthy, unavailable, blackhole |
 | `info` | blue | informational notes |
-| `neutral` | grey `#9AA1A8` | terminated (muted text), and any unmapped state |
+| `neutral` | grey `#9AA1A8` | gone or absent (terminated, deleted, no) in muted text, and any unmapped state |
 
 `stopped` is danger because EC2 reads it that way, although RDS treats it as a resting state; splitting the map by service is its own change.
 
 `StateBadge` (`src/components/state-badge.tsx`) is the only mapping from an AWS state to a tone, and it exposes the result as `data-tone` for tests.
+Its tones are `success`, `progress` (the yellow `warning` token), `caution`, `danger`, `muted` and `neutral`.
+A `progress` state carries the submit button's spinning loader, so a launching instance reads as work under way; `StatusBadge` does not, because its yellow can mean waiting on the user, as a zone's `pending nameservers` does.
+It shows every state lowercase with hyphens, as EC2 writes them, so EKS's `ACTIVE` reads `active` and Bedrock's `IN_PROGRESS` reads `in-progress`; the API value, filters and CLI panels keep AWS's case.
+A good or settled state is always green, never grey, and a status that is not an API state, such as a cluster's health, uses `StatusBadge` with a tone rather than a `Badge`.
+The orange of `caution` is kept clear of the primary accent's hue, which marks actions and never a status.
+`Badge`'s default variant is the brand primary and is never used for a status; a label that is not a status, such as `default v3` or `Leader`, uses a `secondary` or `outline` `Badge`.
 Every status badge measures above 4.8:1 against its tinted background in both themes.
 
 **Danger is separated from the primary accent by hue alone in dark mode**: the two have the same lightness.
@@ -67,18 +74,21 @@ Colour is never the only key: label every series.
 | UI text, tables and forms | Geist (`font-sans`) |
 | IDs, ARNs, IPs, CIDRs, code and eyebrow labels | JetBrains Mono (`font-mono`), with `tabular-nums` for figures |
 
-| Step | Size | Use |
-|---|---|---|
-| `2xl` | 26px | page title (`PageHeading`) |
-| `xl` | 22px | section heading |
-| `lg` | 17px | |
-| `base` | 15px | |
-| `sm` | 14px | body and table default |
-| `xs` | 13px | badges, secondary text |
-| `2xs` | 12px | mono eyebrow labels |
+| Step | rem | At the default 16px browser font | Use |
+|---|---|---|---|
+| `2xl` | 1.625 | 28.6px | page title (`PageHeading`) |
+| `xl` | 1.375 | 24.2px | section heading |
+| `lg` | 1.0625 | 18.7px | |
+| `base` | 0.9375 | 16.5px | |
+| `sm` | 0.875 | 15.4px | body and table default |
+| `xs` | 0.8125 | 14.3px | badges, secondary text |
+| `2xs` | 0.75 | 13.2px | mono eyebrow labels |
 
-The scale sits a step above Tailwind's defaults, as consoles such as Vultr's do, because 13px body text read as cramped on a wide screen.
-Code blocks are set at 12px (`--text-code-block`) so a long command still fits.
+The root font size is 110% (`html` in `src/styles.css`), so one rem is 17.6px at the browser default.
+Everything is sized in rem, so text, spacing and breakpoints scale together, as they would at 110% browser zoom, and a user's own font-size setting still applies.
+The scale sits above Tailwind's defaults, as consoles such as Vultr's do, because the console read as small on a wide screen at 100%.
+Code blocks are set at 0.75rem (`--text-code-block`) so a long command still fits.
+Size new things in rem through the type and spacing scales, never in px, or they will not scale with the rest.
 
 ### Primary and contrast
 
@@ -169,6 +179,22 @@ Every other list is a `DataTable`, and the test also fails when an allowed file
 | S13 | a component outside `ui/` renders a raw `<input>`, `<select>` or `<textarea>` |
 | S15 | a route imports the `ui/alert` primitive instead of using `ErrorBanner` |
 
+Rules that ADR-0016 does not yet number are named by what they hold until their amendment is accepted:
+
+| Rule | Fails when |
+|---|---|
+| status pills come from StateBadge or StatusBadge | a component outside `ui/` renders `<Badge>` with the default variant |
+| status pills come from StateBadge or StatusBadge | a file outside `state-badge.tsx` builds a `rounded-full` pill on a status colour |
+| floating surfaces follow the theme | a file fills with `bg-foreground` or `text-background`, or a tooltip, popover, hover card, menu, select or combobox is not on `bg-popover` and `text-popover-foreground` |
+| expanding a panel keeps its tab | a `fullPage` link does not pass `search={{ tab }}` |
+| resource tags are edited in place | `PanelTags` or `TagsEditor` comes back |
+| lists filter through one FilterBar | a component other than `FilterBar` imports `ui/toggle-group` |
+| a component handed a form reads errors through `useFormState` | a file typed with `UseFormReturn` reads `form.formState` |
+
+The last one matters because the console builds with the React compiler and the unit tests run without it.
+With the compiler on, a child handed a react-hook-form object is cached on that object, whose identity never changes, so errors read from `form.formState` never draw and the submit looks dead.
+`useFormState({ control })` subscribes the child itself.
+
 An exception is named in that rule's allowlist with its reason, and the test fails when an allowed file stops needing it.
 S11, S14 and S16 are held by review, since none has a shape in the source a test can read without false alarms.
 
@@ -177,7 +203,7 @@ S11, S14 and S16 are held by review, since none has a shape in the source a test
 `src/lib/nav.ts` is the one description of the console's navigation.
 The sidebar, the command palette and the breadcrumbs all read it, and `nav.test.ts` checks every destination against the generated route tree, so a renamed route fails a test rather than leaving a dead link.
 
-- **Sidebar.** Groups take generic names rather than AWS's service marks: Compute, Databases, Authentication and Object Storage, not EC2, RDS, IAM and S3. Pages and the rest of the console keep the AWS names, and each group's `service` keeps its AWS name findable in the command palette, where typing it exactly ranks that group first. Each service group folds, and a folded group stays folded across reloads (`localStorage["spinifex-ui-nav-collapsed"]`). Navigating into a folded group unfolds it so the active item shows, and it can then be folded again, the current group included. On a phone the sidebar is a sheet, and any navigation closes it.
+- **Sidebar.** Groups take generic names rather than AWS's service marks: Compute, Databases, Authentication and Object Storage, not EC2, RDS, IAM and S3. Pages and the rest of the console keep the AWS names, and each group's `service` keeps its AWS name findable in the command palette, where typing it exactly ranks that group first. Each service group folds, and a folded group stays folded across reloads (`localStorage["spinifex-ui-nav-collapsed"]`). Navigating into a folded group unfolds it so the active item shows, and it can then be folded again, the current group included. Folding animates through the `nav` variant of `CollapsibleContent`, opening over 200ms and closing over 150ms, and switches at once under reduced motion. On a phone the sidebar is a sheet, and any navigation closes it.
 - **Command palette.** ⌘K or Ctrl K, or the Search button in the header. It lists every page the user can reach, every create flow (`create` on a nav item), and a theme toggle. It is navigation only, with no resource search yet.
 - **Breadcrumbs.** Service, then list page, then the resource or the create flow. A resource ID is set in mono. On a phone only the last crumb shows.
 - **Sign in.** The sign-in card sits over the shell on a light, barely blurred overlay rather than on a page of its own, so the console stays in view. The shell behind it is `inert`, which keeps pointer, keyboard and screen reader out of it until the user signs in.
@@ -202,8 +228,13 @@ So columns drop out when a side panel narrows the list as well as on a small scr
 List pages keep their filter, state and sort in the URL, so a filtered view can be bookmarked, shared and survives a reload.
 EC2 Instances is the reference page (`ec2/(instances)/-components/instances-page.tsx`):
 
-- The first column names the resource: the `Name` tag in the primary colour, with the ID in mono beneath it, opening its side panel.
+- The first column is the instance ID in mono, opening its side panel, and the second its `Name` tag.
+- A long value ends in an ellipsis and shows in full on hover (`TruncatedText`), so it never widens its column.
+- A time column shows a relative time, `33 min ago` or `8 months ago`, with the full timestamp on hover (`RelativeTime`).
+- A Tags column shows one tag key as a pill and `+N` for the rest, and hover lists every tag with its value (`TagPills`); it leaves out `Name`, which has its own column.
+- Below `md`, only the ID, Name and State columns remain.
 - Bulk actions appear above the table once a row is selected. Each action counts only the selected instances it applies to, so a mixed selection never sends a call that must fail, and Terminate asks first and lists every ID.
+- Search and state filters sit in one `FilterBar` (`src/components/filter-bar.tsx`) above the table: a search box, an optional set of choices with counts, and Clear filters while anything is filtered.
 - An empty account gets a call to action, and an empty filter result says the filters are why.
 
 VPCs and Internet gateways follow the same pattern.
@@ -219,6 +250,12 @@ Every list with a delete action has row checkboxes. Selecting a row shows `Selec
 - **The bulk delete and the panel's delete are the same function.** Where a single delete is several calls, such as removing dependants first, the sequence lives in one place and both use it.
 - The verb follows the AWS console: Release for an Elastic IP, Deregister for an image.
 
+### Lists inside a panel
+
+A list inside a panel tab, such as a hosted zone's records, is a `DataTable` with row checkboxes, and a click on a row opens it for editing in a `FormPanel`.
+Rows the user cannot change, such as a zone's own SOA and NS, are muted and cannot be selected (`isRowActionable`).
+A batch delete uses the service's batch call where it has one, so Route 53 removes the whole selection in one `ChangeResourceRecordSets`.
+
 ### Typed confirmation
 
 A delete that cannot be walked back asks the user to type a word first, as the AWS console does before deleting a volume.
@@ -264,13 +301,24 @@ Modelled on UniFi's console: a click on a row opens the resource in a panel besi
 - **URL.** The panel is search state on the list route: `selected=<id>`, `tab=overview|monitoring|settings`, and `create=true`. Defaults are stripped, so a closed panel leaves the plain list address. Opening or closing a panel pushes a history entry, so Back closes it; a tab or filter change replaces the entry.
 - **Layout.** From `lg` (1024px) the panel is fixed to the right edge, full height below the header, flush with no outer margin, and it slides in over the list. The list keeps its width and columns underneath, so nothing reflows when a panel opens, and a soft gradient on the panel's left edge reads as the panel lying on top. Below `lg` there is no room to share, so the panel replaces the list as a full page with a back arrow. `useWideLayout` decides, through `matchMedia`.
 - **`DetailPanel`.** A header with the resource's own page (left) and close (right), the title and its ID, a summary of state and quick actions, then icon tabs with tooltips. Escape closes it unless a dialog or menu is open over it. Closing slides the panel out over 150ms, or at once under reduced motion. Each tab, and a create form, sits in its own error boundary, so a failed query shows in the panel and the list keeps working.
-- **Contents.** `PanelSection` is a folding card, `PanelFields` and `PanelField` a label and value list, with `copy` for a one-click copy of IDs and addresses, and `PanelTags` the tag list. Create forms take an `onClose(id?)` prop: embedded, they hide their back link and heading, and a create selects the new resource.
+- **Contents.** `PanelSection` is a folding card, `PanelFields` and `PanelField` a label and value list, with `copy` for a one-click copy of IDs and addresses, and `InlineTags` the tag list. Create forms take an `onClose(id?)` prop: embedded, they hide their back link and heading, and a create selects the new resource.
 - **One component is both the panel and the page.** Each resource has a single details component (`InstanceDetails`, `VpcDetails`, `InternetGatewayDetails`) that takes `layout="auto" | "page"`. The list renders it as the panel; `/ec2/describe-instances/$id` and its kind render the same component as a page, as one centred column under a standard left-aligned heading, with state and actions on the right and text tabs. A feature added to the panel is on the page by construction, and the page ignores Escape and has a back arrow in place of close. Expanding the panel to its page keeps the tab, and query results such as fetched console output come along from the cache.
 - **A click on the list's background closes the panel**, as UniFi's does. A row selects another resource instead, and links, buttons and inputs keep their own behaviour. Selecting rows for a bulk action closes it too, so the bulk bar and the panel never compete.
 - **Actions.** State actions sit in the summary as a button group, and the rest go in an Actions menu, Unikraft style. Stop, Reboot and every destructive action confirm in a dialog.
 - **Loading.** A panel or page still loading shows `PanelSpinner`, a centred spinner on the background, not a grey skeleton block.
 - **`DetailCard`, `DetailRow` and `ListCard` are retired.** Every resource page uses the panel sections instead.
 
+### Tags
+
+`InlineTags` (`src/components/inline-tags.tsx`) shows a resource's tags in its panel and edits them in place.
+Each row has a pencil, which turns the key and value into inputs and becomes a tick to save, and a bin to remove the tag; `+ Add tag` appends a row being edited.
+Enter saves and Escape cancels the row without closing the panel.
+Every row saves on its own request through the service's own tagging call, and a failure shows beside its row.
+AWS has no rename, so changing a key puts the new key and then removes the old one, and a failure between the two leaves both rather than neither.
+Keys are 1 to 128 characters with no `aws:` prefix and no duplicates, and values up to 256.
+Without `onPut` and `onRemove` it is read-only, as for a system image or an AWS-managed policy.
+Panels use the per-service wrappers in `src/components/resource-tags.tsx` and the writers in `src/mutations/tags.ts` rather than wiring the calls themselves.
+
 ### Forms from a resource's panel
 
 A form that acts on an open resource, such as attaching a volume, scaling a node group or adding a listener, opens in `FormPanel` (`src/components/form-panel.tsx`), never in a dialog.
@@ -285,8 +333,16 @@ Every create flow opens in the side panel over its list, through `CreatePanel`,
 
 - **URL.** `create=true` opens it, plus a list-local field where the flow needs context (`volume` for a snapshot, `instance` for an RDS snapshot, `launch=service|task` on ECS clusters). The old `create-*` addresses redirect to the list with the panel open, so links and bookmarks keep working.
 - **Form.** The fields come in the order AWS asks for them. Settings that are on or off are `Switch`es, and a multi-select picker (subnets, security groups) is a `SwitchList`, a boxed list with one switch per row. Checkboxes only select table rows.
-- **CLI.** Every create form ends with a `CliCommandPanel` showing the AWS CLI commands that send the same request, built from the form's live values and only the flags the console actually sends.
-- **Submit.** `FormActions` turns the submit button into a spinner with its pending label in place, and `FormLock`, a native `<fieldset disabled>`, disables every control above it until the request settles. Nothing above the form appears or moves, and the panel does not scroll.
+- **CLI.** Every create form ends with a `CliCommandPanel` showing the AWS CLI commands that send the same request, built from the form's live values and only the flags the console actually sends. The commands wrap to the panel's width and never scroll sideways; Copy takes the unwrapped text.
+- **Sections.** Related fields are grouped in `PanelSection` boxes, as the instance launch form groups Image, Instance, Network and Root volume.
+- **Nothing to pick, no field.** An optional picker with nothing usable to choose, such as Launch Template or Placement Group with none available, is left out of the form rather than shown holding only `none`.
+- **Advanced settings.** Settings most users leave at their defaults sit in a collapsed Advanced settings section, which opens by itself when a field inside it fails validation.
+- **Choices with a mark.** A few named products, such as database engines or distributions, are `ChoiceTile`s with the product's own mark from `src/assets/`, not a select. One that cannot be used stays visible and disabled, saying why.
+- **Durations and windows.** A duration is a select of presets with the AWS default first and named (`Auto (5 min)` for a record's TTL), keeping a custom value the API returned. A time window, such as an RDS backup or maintenance window, is a day, start time and length picker that refuses an overlap before submitting.
+- **Labels by type.** Where one field means different things by type, its label follows the type, as a record's value reads `IPv4 address` for A and `Mail server` for MX.
+- **Long pickers.** A `SwitchList` that can run long, such as security groups, scrolls inside its box rather than pushing the form down.
+- **Help.** A field's longer explanation goes in an (i) beside its label, through the `info` prop of `TextField` and `SelectField`, rather than in text under the field. Text under a field is for what the user must know before submitting.
+- **Submit.** `FormActions` cross-fades the submit label into a spinner with its pending label over 150ms, at one width, so the button never changes size, and `FormLock`, a native `<fieldset disabled>`, disables every control above it until the request settles. Nothing above the form appears or moves, and the panel does not scroll.
 - **Failure.** A refused request shows beside the button through `FormActions`'s `error`, and the form unlocks. The handler wraps only the mutation call in `try`, with an empty `catch` that returns, because react-hook-form rethrows whatever the handler throws and the error is already on screen from the mutation's state.
 - **Success.** `onClose(id)` closes the panel and selects the new resource, where the list has a panel to select it in.
 - **Appliance images.** A create that needs an ECS, EKS or RDS appliance image shows the missing-image callout inside its panel, never on the list, and only the step that needs the image is blocked.
@@ -306,6 +362,7 @@ A gap of more than three sample periods breaks the line rather than drawing acro
 ## Dates
 
 `formatDateTime` writes the AWS console's form in the viewer's zone, `2026/06/04 02:00 GMT-5`, with a 24-hour clock and UTC as plain `GMT`.
+In a table the time is relative (`relativeTime` in `src/lib/relative-time.ts`), rounded down to the largest whole unit, and the full form shows on hover.
 
 ## Testing
 

scripts/build-lambda-runtime-microvm-image.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+# build-lambda-runtime-microvm-image.sh — build the Lambda runtime microVM kernel
+# + initramfs: a guest running the Lambda guest agent, with virtio-net for its
+# host control connection. It is not shipped.
+#
+# Inputs:
+#   $REPO_ROOT/build/microvm/lambda-runtime/{init.sh,inittab,run-agent}
+#   $REPO_ROOT/bin/lambda-agent  (make bin/lambda-agent)
+#
+# Outputs:
+#   $REPO_ROOT/build/microvm/lambda-runtime/vmlinuz
+#   $REPO_ROOT/build/microvm/lambda-runtime/initramfs.cpio.gz
+#
+# Rootfs strategy and module stripping: scripts/lib/microvm-image.sh.
+# shellcheck disable=SC2034 # variables are consumed by the sourced library
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+BUILD_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
+INIT_SH="$BUILD_DIR/init.sh"
+INITTAB="$BUILD_DIR/inittab"
+RUN_AGENT="$BUILD_DIR/run-agent"
+LAMBDA_AGENT_BIN="$REPO_ROOT/bin/lambda-agent"
+TAG="[build-lambda-runtime-microvm-image]"
+
+ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
+ALPINE_MIRROR="${ALPINE_MIRROR:-https://dl-cdn.alpinelinux.org/alpine}"
+# kmod: the image's own depmod/modprobe rebuild module metadata. busybox ships ip.
+ALPINE_PACKAGES="busybox linux-virt kmod ca-certificates"
+# tiny_power_button turns the ACPI power button into a signal busybox init
+# handles as poweroff.
+MICROVM_EXTRA_MODULES="${MICROVM_EXTRA_MODULES:-tiny_power_button}"
+# virtio_net needs failover and net_failover, which the virtio* glob misses.
+MICROVM_EXTRA_KEEP_GLOBS=("*failover*.ko*")
+MICROVM_GATE_MODULES="qemu_fw_cfg virtio_net"
+
+# shellcheck source=lib/microvm-image.sh
+source "$REPO_ROOT/scripts/lib/microvm-image.sh"
+
+echo "$TAG repo root: $REPO_ROOT"
+echo "$TAG build dir: $BUILD_DIR"
+
+microvm_check_tools
+
+for f in "$INIT_SH" "$INITTAB" "$RUN_AGENT"; do
+    if [ ! -f "$f" ]; then
+        echo "ERROR: $f not found — cannot build initramfs" >&2
+        exit 1
+    fi
+done
+if [ ! -x "$LAMBDA_AGENT_BIN" ]; then
+    echo "ERROR: $LAMBDA_AGENT_BIN not found — run 'make bin/lambda-agent'" >&2
+    exit 1
+fi
+if ! file -b "$LAMBDA_AGENT_BIN" | grep -q "statically linked"; then
+    echo "ERROR: $LAMBDA_AGENT_BIN is not statically linked — it cannot run on the musl guest" >&2
+    exit 1
+fi
+
+microvm_init_dirs
+microvm_make_rootfs
+microvm_empty_dev
+
+echo "$TAG installing init, inittab, run-agent and lambda-agent..."
+install -m 0755 "$INIT_SH" "$CHROOT_DIR/init"
+install -m 0644 "$INITTAB" "$CHROOT_DIR/etc/inittab"
+mkdir -p "$CHROOT_DIR/usr/local/lib/lambda-runtime" "$CHROOT_DIR/usr/local/bin"
+install -m 0755 "$RUN_AGENT" "$CHROOT_DIR/usr/local/lib/lambda-runtime/run-agent"
+install -m 0755 "$LAMBDA_AGENT_BIN" "$CHROOT_DIR/usr/local/bin/lambda-agent"
+# Function-code mountpoint and scratch space ship empty.
+mkdir -p "$CHROOT_DIR/var/task" "$CHROOT_DIR/tmp" \
+    "$CHROOT_DIR/proc" "$CHROOT_DIR/sys" "$CHROOT_DIR/dev" "$CHROOT_DIR/etc/ssl/certs"
+# The bootstrap runs unprivileged, so / must be traversable (mktemp -d makes it
+# 0700) and nothing it runs from may be group-writable.
+chmod 0755 "$CHROOT_DIR" "$CHROOT_DIR/var/task" "$CHROOT_DIR/usr/local/lib/lambda-runtime"
+
+microvm_locate_kernel
+microvm_strip_modules
+microvm_strip_rootfs
+microvm_stage_kernel
+microvm_pack_initramfs
+
+# Nothing reaches $BUILD_DIR until the staged pair passes these checks.
+for out in vmlinuz initramfs.cpio.gz; do
+    if [ ! -s "$STAGE_DIR/$out" ]; then
+        echo "ERROR: staged $out is missing or empty" >&2
+        exit 1
+    fi
+done
+if ! gzip -t "$STAGE_DIR/initramfs.cpio.gz"; then
+    echo "ERROR: staged initramfs.cpio.gz is not valid gzip" >&2
+    exit 1
+fi
+mv -f "$STAGE_DIR/vmlinuz" "$BUILD_DIR/vmlinuz"
+mv -f "$STAGE_DIR/initramfs.cpio.gz" "$BUILD_DIR/initramfs.cpio.gz"
+echo ""
+echo "$TAG done."

scripts/build-sqs-store-microvm-image.sh

@@ -18,6 +18,7 @@
 #   1. apk available on host → install directly into a temp chroot (Alpine CI/CD)
 #   2. docker available      → run alpine container, export filesystem
 #   3. podman available      → same via podman
+# shellcheck disable=SC2034 # variables are consumed by the sourced library
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
@@ -44,21 +45,19 @@ ALPINE_PACKAGES="busybox linux-virt iproute2 ca-certificates e2fsprogs kmod"
 # button (QMP system_powerdown) into a signal busybox init handles as poweroff.
 MICROVM_EXTRA_MODULES="${MICROVM_EXTRA_MODULES:-ext4 tiny_power_button}"
 
+# Module file globs kept on top of virtio*/*fw_cfg*: virtio_net deps for live migration.
+MICROVM_EXTRA_KEEP_GLOBS=("*failover*.ko*")
+# Modules the image modprobe must resolve, beyond the extras.
+MICROVM_GATE_MODULES="qemu_fw_cfg virtio_net"
+
+# shellcheck source=lib/microvm-image.sh
+source "$REPO_ROOT/scripts/lib/microvm-image.sh"
+
 echo "$TAG repo root: $REPO_ROOT"
 echo "$TAG build dir: $BUILD_DIR"
 mkdir -p "$BUILD_DIR"
 
-# --- Verify non-negotiable tools ---
-# fakeroot is mandatory: device nodes are written into the cpio archive inside a
-# faked-root environment so no real device node is ever created on the host
-# filesystem (which previously required sudo mknod and risked clobbering host
-# /dev/null on cleanup).
-for tool in cpio gzip find fakeroot; do
-    if ! command -v "$tool" >/dev/null 2>&1; then
-        echo "ERROR: required tool not found: $tool" >&2
-        exit 1
-    fi
-done
+microvm_check_tools
 
 if [ ! -x "$SQS_WORKER_BIN" ]; then
     echo "ERROR: $SQS_WORKER_BIN not found — run 'make bin/sqs-worker'" >&2
@@ -90,100 +89,11 @@ if [ "$nats_server_version" != "$NATS_SERVER_PIN" ]; then
 fi
 echo "$TAG nats-server: $nats_server_version (matches go.mod)"
 
-# --- Create chroot ---
-CHROOT_DIR=$(mktemp -d)
-# Guard: every destructive op below targets "$CHROOT_DIR/...". An empty or
-# unexpected value would aim rm -rf at the host (e.g. /dev). Refuse anything that
-# is not a freshly-created temp dir.
-case "${CHROOT_DIR:?mktemp -d returned empty}" in
-    /tmp/*|/var/tmp/*|"${TMPDIR:-/nonexistent}"/*) ;;
-    *) echo "ERROR: refusing to use unexpected CHROOT_DIR: $CHROOT_DIR" >&2; exit 1 ;;
-esac
-# Outputs are written here first, beside $BUILD_DIR so publishing is a rename.
-STAGE_DIR=$(mktemp -d "$BUILD_DIR/.stage.XXXXXX")
-CONTAINER_TOOL=""
-CONTAINER_CID=""
-cleanup() {
-    echo "$TAG cleaning up chroot: $CHROOT_DIR"
-    rm -rf "$CHROOT_DIR"
-    rm -rf "${STAGE_DIR:?}"
-    if [ -n "$CONTAINER_CID" ] && [ -n "$CONTAINER_TOOL" ]; then
-        echo "$TAG cleaning up container: $CONTAINER_CID"
-        $CONTAINER_TOOL rm -f "$CONTAINER_CID" >/dev/null 2>&1 || true
-    fi
-}
-trap cleanup EXIT
+microvm_init_dirs
 
-# --- Populate rootfs ---
-# Strategy 1: native apk (Alpine hosts, CI)
-if command -v apk >/dev/null 2>&1; then
-    echo "$TAG rootfs: native apk"
-    mkdir -p \
-        "$CHROOT_DIR/etc/apk" \
-        "$CHROOT_DIR/lib/apk/db" \
-        "$CHROOT_DIR/var/cache/apk"
-    cat > "$CHROOT_DIR/etc/apk/repositories" <<EOF
-${ALPINE_MIRROR}/v${ALPINE_VERSION}/main
-${ALPINE_MIRROR}/v${ALPINE_VERSION}/community
-EOF
-    apk add \
-        --root "$CHROOT_DIR" \
-        --arch x86_64 \
-        --no-cache \
-        --allow-untrusted \
-        --initdb \
-        $ALPINE_PACKAGES
+microvm_make_rootfs
 
-# Strategy 2: docker -> BuildKit local export.
-# Do NOT use `docker export` here. On Docker's overlay2 driver it leaks the
-# container's overlay mount (the mount survives `docker rm`), orphaning layers
-# under /var/lib/docker that only a daemon restart reclaims; on a busy CI host
-# this grows without bound. `buildx --output type=local` writes the image
-# filesystem straight to a directory and releases every mount, so nothing leaks.
-elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
-    echo "$TAG rootfs: docker buildx --output type=local (alpine:${ALPINE_VERSION})"
-    BUILD_CTX=$(mktemp -d)
-    cat > "$BUILD_CTX/Dockerfile" <<DOCKERFILE
-FROM alpine:${ALPINE_VERSION}
-RUN apk add --no-cache ${ALPINE_PACKAGES}
-DOCKERFILE
-    docker buildx build \
-        --platform linux/amd64 \
-        --output "type=local,dest=${CHROOT_DIR}" \
-        "$BUILD_CTX"
-    rm -rf "$BUILD_CTX"
-
-# Strategy 3: podman -> run + export. Podman's store does not exhibit the docker
-# overlay2 export-mount leak, and podman has no buildx.
-elif command -v podman >/dev/null 2>&1; then
-    echo "$TAG rootfs: podman export (alpine:${ALPINE_VERSION})"
-    CONTAINER_TOOL=podman
-    CONTAINER_CID=$(podman run -d \
-        "alpine:${ALPINE_VERSION}" \
-        sh -c "apk add --no-cache ${ALPINE_PACKAGES}")
-
-    exit_code=$(podman wait "$CONTAINER_CID")
-    if [ "$exit_code" != "0" ]; then
-        echo "ERROR: package installation failed in container (exit $exit_code)" >&2
-        exit 1
-    fi
-
-    podman export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR"
-
-else
-    echo "ERROR: no rootfs build tool available." >&2
-    echo "       Install one of: apk (Alpine), docker, podman" >&2
-    exit 1
-fi
-
-# --- /dev: empty mountpoint only ---
-# tar extraction without root silently creates 0-byte regular files for device
-# nodes. Drop them and leave an empty /dev mountpoint. The two static nodes the
-# kernel needs (/dev/console to wire init's stdio, /dev/null) are written into
-# the cpio archive under fakeroot at pack time — NOT created on the host. init
-# mounts devtmpfs over /dev as its first action for everything else at runtime.
-rm -rf "${CHROOT_DIR:?}/dev"
-mkdir "$CHROOT_DIR/dev"
+microvm_empty_dev
 
 # --- Place init script ---
 echo "$TAG installing init.sh as /init..."
@@ -227,30 +137,8 @@ mkdir -p \
     "$CHROOT_DIR/dev" \
     "$CHROOT_DIR/etc/ssl/certs"
 
-# --- Locate kernel ---
-KERNEL_IMG=$(find "$CHROOT_DIR/boot" -name "vmlinuz*" | sort | tail -1)
-if [ -z "$KERNEL_IMG" ]; then
-    echo "ERROR: no vmlinuz found in $CHROOT_DIR/boot" >&2
-    exit 1
-fi
-echo "$TAG kernel image: $KERNEL_IMG"
 
-KERNEL_VER=$(basename "$KERNEL_IMG" | sed 's/vmlinuz-//')
-KERNEL_CONFIG="$CHROOT_DIR/boot/config-${KERNEL_VER}"
-
-# --- Assert fw_cfg module ---
-if ! find "$CHROOT_DIR/lib/modules" \
-        -name "qemu_fw_cfg.ko*" \
-        -o -name "fw_cfg_sysfs.ko*" \
-    | grep -q .; then
-    if ! grep -q "CONFIG_FW_CFG_SYSFS=y" "$KERNEL_CONFIG" 2>/dev/null; then
-        echo "ERROR: qemu_fw_cfg module missing from initramfs — fw_cfg reads will fail at boot" >&2
-        exit 1
-    fi
-    echo "$TAG fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
-else
-    echo "$TAG fw_cfg: module found in lib/modules"
-fi
+microvm_locate_kernel
 
 # --- Assert ext4 userspace tools (data volume) ---
 for fs_tool in mkfs.ext4 e2fsck; do
@@ -261,227 +149,10 @@ for fs_tool in mkfs.ext4 e2fsck; do
 done
 echo "$TAG e2fsprogs: mkfs.ext4 + e2fsck present"
 
-# --- Strip kernel modules (Phase 2 keep-list approach) ---
-# Only modules needed by a virtio-mmio microvm guest are kept:
-#   virtio*.ko*    — virtio bus, net, blk, rng, console, and helpers
-#   *fw_cfg*.ko*   — QEMU fw_cfg sysfs driver (delivers boot blobs)
-#   *failover*.ko* — net_failover + failover (virtio_net deps for live migration)
-#   $MICROVM_EXTRA_MODULES — ext4 by default, plus dependency closure
-# All other loadable modules are removed. Drivers compiled into the kernel
-# (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
-
-# module_path prints a module's kernel/ path from the stock modules.dep, by
-# name or modules.alias; it fails when the module is not loadable.
-module_path() {
-    local kdir="$1" name="${2//-/_}" path alias_target
-    path=$(awk -F: -v n="$name" '{ f = $1; sub(/.*\//, "", f); sub(/\.ko(\.gz|\.xz|\.zst)?$/, "", f);
-        gsub(/-/, "_", f); if (f == n) { print $1; exit } }' "$kdir/modules.dep")
-    if [ -z "$path" ] && [ -f "$kdir/modules.alias" ]; then
-        alias_target=$(awk -v n="$2" '$1 == "alias" && $2 == n { print $3; exit }' "$kdir/modules.alias")
-        [ -n "$alias_target" ] && [ "$alias_target" != "$2" ] && path=$(module_path "$kdir" "$alias_target")
-    fi
-    [ -n "$path" ] && printf '%s\n' "$path"
-}
-
-# module_builtin reports whether a module name is compiled into the kernel.
-module_builtin() {
-    local kdir="$1" name="${2//-/_}"
-    [ -f "$kdir/modules.builtin" ] || return 1
-    awk -v n="$name" '{ f = $0; sub(/.*\//, "", f); sub(/\.ko$/, "", f); gsub(/-/, "_", f);
-        if (f == n) { found = 1; exit } } END { exit !found }' "$kdir/modules.builtin"
-}
-
-# image_kmod runs the image's own kmod (depmod, modprobe) through its musl
-# loader, so module metadata matches the guest's modprobe without a host kmod.
-image_kmod() {
-    "$CHROOT_DIR/lib/ld-musl-x86_64.so.1" --library-path "$CHROOT_DIR/lib:$CHROOT_DIR/usr/lib" \
-        --argv0 "$1" "$CHROOT_DIR/bin/kmod" "${@:2}"
-}
-
-echo "$TAG stripping kernel modules to virtio+fw_cfg+${MICROVM_EXTRA_MODULES} only..."
-for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
-    kver="$(basename "$kver_dir")"
-    kernel_dir="$kver_dir/kernel"
-    [ -d "$kernel_dir" ] || continue
-
-    if [ ! -s "$kver_dir/modules.dep" ]; then
-        echo "ERROR: stock ${kver_dir}modules.dep missing — cannot resolve module dependencies" >&2
-        exit 1
-    fi
-
-    # Extra modules start the closure; the glob families are kept as found.
-    keep_list=$(mktemp)
-    : > "$keep_list"
-    for name in $MICROVM_EXTRA_MODULES; do
-        if ! path=$(module_path "$kver_dir" "$name"); then
-            if module_builtin "$kver_dir" "$name"; then continue; fi
-            echo "ERROR: module $name is neither loadable nor built into $kver" >&2
-            exit 1
-        fi
-        echo "$path" >> "$keep_list"
-    done
-
-    # Close over modules.dep hard dependencies and modules.softdep pre:/post:
-    # names until the set stops growing. A softdep must be loadable or built in.
-    while :; do
-        before=$(sort -u "$keep_list" | wc -l)
-        for path in $(sort -u "$keep_list"); do
-            awk -F': ' -v p="$path" '$1 == p { n = split($2, d, " "); for (i = 1; i <= n; i++) print d[i] }' \
-                "$kver_dir/modules.dep" >> "$keep_list"
-            mod=$(basename "$path"); mod="${mod%%.ko*}"; mod="${mod//-/_}"
-            [ -f "$kver_dir/modules.softdep" ] || continue
-            for soft in $(awk -v n="$mod" '{ m = $2; gsub(/-/, "_", m) }
-                    $1 == "softdep" && m == n { for (i = 3; i <= NF; i++) if ($i != "pre:" && $i != "post:") print $i }' \
-                    "$kver_dir/modules.softdep"); do
-                if soft_path=$(module_path "$kver_dir" "$soft"); then
-                    echo "$soft_path" >> "$keep_list"
-                elif ! module_builtin "$kver_dir" "$soft"; then
-                    echo "ERROR: softdep $soft of $mod is neither loadable nor built into $kver" >&2
-                    exit 1
-                fi
-            done
-        done
-        after=$(sort -u "$keep_list" | wc -l)
-        [ "$before" -eq "$after" ] && break
-    done
-
-    extra_names=$(sort -u "$keep_list" | while read -r p; do m=$(basename "$p"); m="${m%%.ko*}"; echo "${m//-/_}"; done)
-    (cd "$kver_dir" && find kernel -name "virtio*.ko*" -o -name "*fw_cfg*.ko*" \
-        -o -name "*failover*.ko*") >> "$keep_list"
-
-    # Copy the keeper set out, preserving each kernel/ relative path.
-    tmp_keep=$(mktemp -d)
-    while read -r rel; do
-        rel="${rel#kernel/}"
-        mkdir -p "$tmp_keep/$(dirname "$rel")"
-        cp "$kernel_dir/$rel" "$tmp_keep/$rel"
-    done < <(sort -u "$keep_list")
-    rm -f "$keep_list"
-
-    # Replace kernel module tree with the keeper set.
-    rm -rf "$kernel_dir"
-    mkdir -p "$kernel_dir"
-    # Restore each kept file into the same relative path.
-    (cd "$tmp_keep" && find . -name "*.ko*" | while read -r f; do
-        dest_dir="$kernel_dir/$(dirname "${f#./}")"
-        mkdir -p "$dest_dir"
-        cp "$f" "$dest_dir/"
-    done)
-    rm -rf "$tmp_keep"
-
-    # Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
-    # kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
-    # produces an EMPTY modules.dep from compressed modules — the guest's
-    # modprobe then no-ops and qemu_fw_cfg never loads. Plain .ko sidesteps the
-    # host-kmod dependency entirely and the guest needs no decompressor either.
-    find "$kernel_dir" -name "*.ko.gz" -exec gunzip -f {} +
-
-    # Independent closure gate: each extra-closure module's own .modinfo depends=
-    # must name a kept module or a built-in, so a resolver bug fails here.
-    kept_names=$(find "$kernel_dir" -name "*.ko" -exec basename {} .ko \; | tr '-' '_' | sort -u)
-    while read -r ko; do
-        ko_name=$(basename "$ko" .ko); ko_name="${ko_name//-/_}"
-        grep -qx "$ko_name" <<< "$extra_names" || continue
-        deps=$(tr '\0' '\n' < "$ko" | grep -a -m1 '^depends=' | cut -d= -f2 | tr ',' ' ' || true)
-        for dep in $deps; do
-            dep="${dep//-/_}"
-            if ! grep -qx "$dep" <<< "$kept_names" && ! module_builtin "$kver_dir" "$dep"; then
-                echo "ERROR: $(basename "$ko") depends on $dep, which is neither kept nor built in" >&2
-                exit 1
-            fi
-        done
-    done < <(find "$kernel_dir" -name "*.ko")
-    for name in $MICROVM_EXTRA_MODULES; do
-        if ! grep -qx "${name//-/_}" <<< "$kept_names" && ! module_builtin "$kver_dir" "$name"; then
-            echo "ERROR: required module $name missing from the stripped module tree" >&2
-            exit 1
-        fi
-    done
-    echo "$TAG extra module closure: $(echo "$extra_names" | tr '\n' ' ')"
-    echo "$TAG modules kept: $(echo "$kept_names" | tr '\n' ' ')"
-
-    # Rebuild module metadata with the image's own kmod, the binary the guest's
-    # modprobe runs, so no host depmod is needed. Stock indexes still name .ko.gz
-    # files, so they are removed first and a failed rebuild cannot ship them.
-    for f in dep dep.bin alias alias.bin softdep weakdep symbols symbols.bin \
-        builtin.bin builtin.alias.bin devname; do
-        rm -f "$kver_dir/modules.$f"
-    done
-    if ! image_kmod depmod -b "$CHROOT_DIR" "$kver"; then
-        echo "ERROR: image depmod failed for $kver" >&2
-        exit 1
-    fi
-    if [ ! -s "$kver_dir/modules.dep" ] || [ ! -s "$kver_dir/modules.dep.bin" ]; then
-        echo "ERROR: ${kver_dir}modules.dep or modules.dep.bin missing or empty after depmod" >&2
-        exit 1
-    fi
-    # Resolution gate: the image's modprobe must resolve every required module
-    # to plain .ko files present in the stripped tree.
-    for name in $MICROVM_EXTRA_MODULES qemu_fw_cfg virtio_net; do
-        module_builtin "$kver_dir" "$name" && continue
-        if ! plan=$(image_kmod modprobe -d "$CHROOT_DIR" -S "$kver" --show-depends "$name"); then
-            echo "ERROR: image modprobe cannot resolve $name for $kver" >&2
-            exit 1
-        fi
-        while read -r verb path _; do
-            [ "$verb" = "insmod" ] || continue
-            case "$path" in
-                "$CHROOT_DIR/lib/modules/$kver/kernel/"*.ko) [ -f "$path" ] && continue ;;
-            esac
-            echo "ERROR: modprobe plan for $name names $path, not a kept plain .ko" >&2
-            exit 1
-        done <<< "$plan"
-        echo "$TAG modprobe $name: $(awk '{ n = split($2, a, "/"); printf "%s ", a[n] }' <<< "$plan")"
-    done
-done
-
-# --- Strip package-manager artifacts (not needed at runtime) ---
-echo "$TAG stripping package manager artifacts..."
-rm -rf \
-    "$CHROOT_DIR/lib/apk" \
-    "$CHROOT_DIR/var/cache/apk" \
-    "$CHROOT_DIR/etc/apk" \
-    "$CHROOT_DIR/usr/share/man" \
-    "$CHROOT_DIR/usr/share/doc" \
-    "$CHROOT_DIR/usr/share/apk" \
-    "$CHROOT_DIR/usr/include" \
-    "$CHROOT_DIR/usr/lib/pkgconfig"
-
-# --- Strip debug symbols from binaries ---
-# nats-server in /usr/local/bin is left as built (-s -w), so the shipped bytes
-# are the ones whose version was checked above.
-echo "$TAG stripping debug symbols from binaries..."
-find "$CHROOT_DIR/usr/sbin" "$CHROOT_DIR/usr/bin" \
-     "$CHROOT_DIR/sbin" "$CHROOT_DIR/bin" \
-    -type f 2>/dev/null | while read -r bin; do
-    strip --strip-all "$bin" 2>/dev/null || true
-done
-
-# --- Copy vmlinuz out before stripping /boot from chroot ---
-VMLINUZ_OUT="$STAGE_DIR/vmlinuz"
-echo "$TAG copying kernel: $VMLINUZ_OUT"
-cp "$KERNEL_IMG" "$VMLINUZ_OUT"
-
-# Drop /boot from the chroot — vmlinuz, Alpine's stock initramfs-virt, System.map
-# and kernel config all live here (~25 MiB). None are needed at runtime: the
-# guest already has the kernel loaded by QEMU's -kernel, and Alpine's initramfs
-# is unused because we replace it with our own /init. Leaving /boot in the cpio
-# nearly doubles uncompressed initramfs size and forces a 512 MiB guest just to
-# survive decompression.
-rm -rf "${CHROOT_DIR:?}/boot"
-
-# --- Build initramfs ---
-INITRAMFS_OUT="$STAGE_DIR/initramfs.cpio.gz"
-echo "$TAG building initramfs: $INITRAMFS_OUT"
-# Create the static device nodes and pack the archive inside a single fakeroot
-# session: mknod/cpio see faked device entries and write real char-device
-# records into the cpio, while the host filesystem never gets an actual node.
-fakeroot sh -c '
-    cd "$1" || exit 1
-    mknod -m 600 dev/console c 5 1
-    mknod -m 666 dev/null    c 1 3
-    find . | cpio --quiet -o -H newc | gzip -9 > "$2"
-' _ "$CHROOT_DIR" "$INITRAMFS_OUT"
+microvm_strip_modules
+microvm_strip_rootfs
+microvm_stage_kernel
+microvm_pack_initramfs
 
 # --- Gate the staged pair, then publish it ---
 # Nothing reaches $BUILD_DIR until every gate passes, so a failed build cannot

scripts/lib/microvm-image.sh

@@ -0,0 +1,391 @@
+#!/bin/bash
+# microvm-image.sh — shared steps for the microVM image builders, sourced by
+# build-sqs-store-microvm-image.sh and build-lambda-runtime-microvm-image.sh.
+#
+# The caller sets before use:
+#   TAG                      log prefix
+#   BUILD_DIR                output directory; STAGE_DIR is created beside it
+#   ALPINE_VERSION, ALPINE_MIRROR, ALPINE_PACKAGES
+#   MICROVM_EXTRA_MODULES    modules kept by name, with their dependency closure
+#   MICROVM_GATE_MODULES     modules the image modprobe must resolve, beyond the extras
+#   MICROVM_EXTRA_KEEP_GLOBS bash array of module file globs kept on top of virtio*/*fw_cfg*
+# Functions set CHROOT_DIR, STAGE_DIR, KERNEL_IMG and KERNEL_VER for the caller.
+
+
+microvm_check_tools() {
+    # --- Verify non-negotiable tools ---
+    # fakeroot is mandatory: device nodes are written into the cpio archive inside a
+    # faked-root environment so no real device node is ever created on the host
+    # filesystem (which previously required sudo mknod and risked clobbering host
+    # /dev/null on cleanup).
+    for tool in cpio gzip find fakeroot; do
+        if ! command -v "$tool" >/dev/null 2>&1; then
+            echo "ERROR: required tool not found: $tool" >&2
+            exit 1
+        fi
+    done
+}
+
+microvm_init_dirs() {
+    # --- Create chroot ---
+    CHROOT_DIR=$(mktemp -d)
+    # Guard: every destructive op below targets "$CHROOT_DIR/...". An empty or
+    # unexpected value would aim rm -rf at the host (e.g. /dev). Refuse anything that
+    # is not a freshly-created temp dir.
+    case "${CHROOT_DIR:?mktemp -d returned empty}" in
+        /tmp/*|/var/tmp/*|"${TMPDIR:-/nonexistent}"/*) ;;
+        *) echo "ERROR: refusing to use unexpected CHROOT_DIR: $CHROOT_DIR" >&2; exit 1 ;;
+    esac
+    # Outputs are written here first, beside $BUILD_DIR so publishing is a rename.
+    STAGE_DIR=$(mktemp -d "$BUILD_DIR/.stage.XXXXXX")
+    CONTAINER_TOOL=""
+    CONTAINER_CID=""
+    trap microvm_cleanup EXIT
+}
+
+microvm_cleanup() {
+        echo "$TAG cleaning up chroot: $CHROOT_DIR"
+        rm -rf "$CHROOT_DIR"
+        rm -rf "${STAGE_DIR:?}"
+        if [ -n "$CONTAINER_CID" ] && [ -n "$CONTAINER_TOOL" ]; then
+            echo "$TAG cleaning up container: $CONTAINER_CID"
+            $CONTAINER_TOOL rm -f "$CONTAINER_CID" >/dev/null 2>&1 || true
+        fi
+}
+
+# microvm_make_rootfs populates $CHROOT_DIR from ALPINE_PACKAGES.
+microvm_make_rootfs() {
+    # --- Populate rootfs ---
+    # Strategy 1: native apk (Alpine hosts, CI)
+    if command -v apk >/dev/null 2>&1; then
+        echo "$TAG rootfs: native apk"
+        mkdir -p \
+            "$CHROOT_DIR/etc/apk" \
+            "$CHROOT_DIR/lib/apk/db" \
+            "$CHROOT_DIR/var/cache/apk"
+        cat > "$CHROOT_DIR/etc/apk/repositories" <<EOF
+${ALPINE_MIRROR}/v${ALPINE_VERSION}/main
+${ALPINE_MIRROR}/v${ALPINE_VERSION}/community
+EOF
+        apk add \
+            --root "$CHROOT_DIR" \
+            --arch x86_64 \
+            --no-cache \
+            --allow-untrusted \
+            --initdb \
+            $ALPINE_PACKAGES
+
+    # Strategy 2: docker -> BuildKit local export.
+    # Do NOT use `docker export` here. On Docker's overlay2 driver it leaks the
+    # container's overlay mount (the mount survives `docker rm`), orphaning layers
+    # under /var/lib/docker that only a daemon restart reclaims; on a busy CI host
+    # this grows without bound. `buildx --output type=local` writes the image
+    # filesystem straight to a directory and releases every mount, so nothing leaks.
+    elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
+        echo "$TAG rootfs: docker buildx --output type=local (alpine:${ALPINE_VERSION})"
+        BUILD_CTX=$(mktemp -d)
+        cat > "$BUILD_CTX/Dockerfile" <<DOCKERFILE
+FROM alpine:${ALPINE_VERSION}
+RUN apk add --no-cache ${ALPINE_PACKAGES}
+DOCKERFILE
+        docker buildx build \
+            --platform linux/amd64 \
+            --output "type=local,dest=${CHROOT_DIR}" \
+            "$BUILD_CTX"
+        rm -rf "$BUILD_CTX"
+
+    # Strategy 3: podman -> run + export. Podman's store does not exhibit the docker
+    # overlay2 export-mount leak, and podman has no buildx.
+    elif command -v podman >/dev/null 2>&1; then
+        echo "$TAG rootfs: podman export (alpine:${ALPINE_VERSION})"
+        CONTAINER_TOOL=podman
+        CONTAINER_CID=$(podman run -d \
+            "alpine:${ALPINE_VERSION}" \
+            sh -c "apk add --no-cache ${ALPINE_PACKAGES}")
+
+        exit_code=$(podman wait "$CONTAINER_CID")
+        if [ "$exit_code" != "0" ]; then
+            echo "ERROR: package installation failed in container (exit $exit_code)" >&2
+            exit 1
+        fi
+
+        podman export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR"
+
+    else
+        echo "ERROR: no rootfs build tool available." >&2
+        echo "       Install one of: apk (Alpine), docker, podman" >&2
+        exit 1
+    fi
+}
+
+microvm_empty_dev() {
+    # --- /dev: empty mountpoint only ---
+    # tar extraction without root silently creates 0-byte regular files for device
+    # nodes. Drop them and leave an empty /dev mountpoint. The two static nodes the
+    # kernel needs (/dev/console to wire init's stdio, /dev/null) are written into
+    # the cpio archive under fakeroot at pack time — NOT created on the host. init
+    # mounts devtmpfs over /dev as its first action for everything else at runtime.
+    rm -rf "${CHROOT_DIR:?}/dev"
+    mkdir "$CHROOT_DIR/dev"
+}
+
+# microvm_locate_kernel finds the kernel and asserts fw_cfg is loadable or built in.
+microvm_locate_kernel() {
+    # --- Locate kernel ---
+    KERNEL_IMG=$(find "$CHROOT_DIR/boot" -name "vmlinuz*" | sort | tail -1)
+    if [ -z "$KERNEL_IMG" ]; then
+        echo "ERROR: no vmlinuz found in $CHROOT_DIR/boot" >&2
+        exit 1
+    fi
+    echo "$TAG kernel image: $KERNEL_IMG"
+
+    KERNEL_VER=$(basename "$KERNEL_IMG" | sed 's/vmlinuz-//')
+    KERNEL_CONFIG="$CHROOT_DIR/boot/config-${KERNEL_VER}"
+
+    # --- Assert fw_cfg module ---
+    if ! find "$CHROOT_DIR/lib/modules" \
+            -name "qemu_fw_cfg.ko*" \
+            -o -name "fw_cfg_sysfs.ko*" \
+        | grep -q .; then
+        if ! grep -q "CONFIG_FW_CFG_SYSFS=y" "$KERNEL_CONFIG" 2>/dev/null; then
+            echo "ERROR: qemu_fw_cfg module missing from initramfs — fw_cfg reads will fail at boot" >&2
+            exit 1
+        fi
+        echo "$TAG fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
+    else
+        echo "$TAG fw_cfg: module found in lib/modules"
+    fi
+}
+
+# --- Strip kernel modules (Phase 2 keep-list approach) ---
+# Only modules needed by a virtio-mmio microvm guest are kept:
+#   virtio*.ko*    — virtio bus, net, blk, rng, console, and helpers
+#   *fw_cfg*.ko*   — QEMU fw_cfg sysfs driver (delivers boot blobs)
+#   $MICROVM_EXTRA_KEEP_GLOBS — caller-named module file globs
+#   $MICROVM_EXTRA_MODULES — caller-named modules, plus dependency closure
+# All other loadable modules are removed. Drivers compiled into the kernel
+# (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
+
+# module_path prints a module's kernel/ path from the stock modules.dep, by
+# name or modules.alias; it fails when the module is not loadable.
+module_path() {
+    local kdir="$1" name="${2//-/_}" path alias_target
+    path=$(awk -F: -v n="$name" '{ f = $1; sub(/.*\//, "", f); sub(/\.ko(\.gz|\.xz|\.zst)?$/, "", f);
+        gsub(/-/, "_", f); if (f == n) { print $1; exit } }' "$kdir/modules.dep")
+    if [ -z "$path" ] && [ -f "$kdir/modules.alias" ]; then
+        alias_target=$(awk -v n="$2" '$1 == "alias" && $2 == n { print $3; exit }' "$kdir/modules.alias")
+        [ -n "$alias_target" ] && [ "$alias_target" != "$2" ] && path=$(module_path "$kdir" "$alias_target")
+    fi
+    [ -n "$path" ] && printf '%s\n' "$path"
+}
+
+# module_builtin reports whether a module name is compiled into the kernel.
+module_builtin() {
+    local kdir="$1" name="${2//-/_}"
+    [ -f "$kdir/modules.builtin" ] || return 1
+    awk -v n="$name" '{ f = $0; sub(/.*\//, "", f); sub(/\.ko$/, "", f); gsub(/-/, "_", f);
+        if (f == n) { found = 1; exit } } END { exit !found }' "$kdir/modules.builtin"
+}
+
+# image_kmod runs the image's own kmod (depmod, modprobe) through its musl
+# loader, so module metadata matches the guest's modprobe without a host kmod.
+image_kmod() {
+    "$CHROOT_DIR/lib/ld-musl-x86_64.so.1" --library-path "$CHROOT_DIR/lib:$CHROOT_DIR/usr/lib" \
+        --argv0 "$1" "$CHROOT_DIR/bin/kmod" "${@:2}"
+}
+
+microvm_strip_modules() {
+    echo "$TAG stripping kernel modules to virtio+fw_cfg+${MICROVM_EXTRA_MODULES} only..."
+    for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
+        kver="$(basename "$kver_dir")"
+        kernel_dir="$kver_dir/kernel"
+        [ -d "$kernel_dir" ] || continue
+
+        if [ ! -s "$kver_dir/modules.dep" ]; then
+            echo "ERROR: stock ${kver_dir}modules.dep missing — cannot resolve module dependencies" >&2
+            exit 1
+        fi
+
+        # Extra modules start the closure; the glob families are kept as found.
+        keep_list=$(mktemp)
+        : > "$keep_list"
+        for name in $MICROVM_EXTRA_MODULES; do
+            if ! path=$(module_path "$kver_dir" "$name"); then
+                if module_builtin "$kver_dir" "$name"; then continue; fi
+                echo "ERROR: module $name is neither loadable nor built into $kver" >&2
+                exit 1
+            fi
+            echo "$path" >> "$keep_list"
+        done
+
+        # Close over modules.dep hard dependencies and modules.softdep pre:/post:
+        # names until the set stops growing. A softdep must be loadable or built in.
+        while :; do
+            before=$(sort -u "$keep_list" | wc -l)
+            for path in $(sort -u "$keep_list"); do
+                awk -F': ' -v p="$path" '$1 == p { n = split($2, d, " "); for (i = 1; i <= n; i++) print d[i] }' \
+                    "$kver_dir/modules.dep" >> "$keep_list"
+                mod=$(basename "$path"); mod="${mod%%.ko*}"; mod="${mod//-/_}"
+                [ -f "$kver_dir/modules.softdep" ] || continue
+                for soft in $(awk -v n="$mod" '{ m = $2; gsub(/-/, "_", m) }
+                        $1 == "softdep" && m == n { for (i = 3; i <= NF; i++) if ($i != "pre:" && $i != "post:") print $i }' \
+                        "$kver_dir/modules.softdep"); do
+                    if soft_path=$(module_path "$kver_dir" "$soft"); then
+                        echo "$soft_path" >> "$keep_list"
+                    elif ! module_builtin "$kver_dir" "$soft"; then
+                        echo "ERROR: softdep $soft of $mod is neither loadable nor built into $kver" >&2
+                        exit 1
+                    fi
+                done
+            done
+            after=$(sort -u "$keep_list" | wc -l)
+            [ "$before" -eq "$after" ] && break
+        done
+
+        extra_names=$(sort -u "$keep_list" | while read -r p; do m=$(basename "$p"); m="${m%%.ko*}"; echo "${m//-/_}"; done)
+        find_args=(-name "virtio*.ko*" -o -name "*fw_cfg*.ko*")
+        for glob in ${MICROVM_EXTRA_KEEP_GLOBS[@]+"${MICROVM_EXTRA_KEEP_GLOBS[@]}"}; do
+            find_args+=(-o -name "$glob")
+        done
+        (cd "$kver_dir" && find kernel "${find_args[@]}") >> "$keep_list"
+
+        # Copy the keeper set out, preserving each kernel/ relative path.
+        tmp_keep=$(mktemp -d)
+        while read -r rel; do
+            rel="${rel#kernel/}"
+            mkdir -p "$tmp_keep/$(dirname "$rel")"
+            cp "$kernel_dir/$rel" "$tmp_keep/$rel"
+        done < <(sort -u "$keep_list")
+        rm -f "$keep_list"
+
+        # Replace kernel module tree with the keeper set.
+        rm -rf "$kernel_dir"
+        mkdir -p "$kernel_dir"
+        # Restore each kept file into the same relative path.
+        (cd "$tmp_keep" && find . -name "*.ko*" | while read -r f; do
+            dest_dir="$kernel_dir/$(dirname "${f#./}")"
+            mkdir -p "$dest_dir"
+            cp "$f" "$dest_dir/"
+        done)
+        rm -rf "$tmp_keep"
+
+        # Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
+        # kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
+        # produces an EMPTY modules.dep from compressed modules — the guest's
+        # modprobe then no-ops and qemu_fw_cfg never loads. Plain .ko sidesteps the
+        # host-kmod dependency entirely and the guest needs no decompressor either.
+        find "$kernel_dir" -name "*.ko.gz" -exec gunzip -f {} +
+
+        # Independent closure gate: each extra-closure module's own .modinfo depends=
+        # must name a kept module or a built-in, so a resolver bug fails here.
+        kept_names=$(find "$kernel_dir" -name "*.ko" -exec basename {} .ko \; | tr '-' '_' | sort -u)
+        while read -r ko; do
+            ko_name=$(basename "$ko" .ko); ko_name="${ko_name//-/_}"
+            grep -qx "$ko_name" <<< "$extra_names" || continue
+            deps=$(tr '\0' '\n' < "$ko" | grep -a -m1 '^depends=' | cut -d= -f2 | tr ',' ' ' || true)
+            for dep in $deps; do
+                dep="${dep//-/_}"
+                if ! grep -qx "$dep" <<< "$kept_names" && ! module_builtin "$kver_dir" "$dep"; then
+                    echo "ERROR: $(basename "$ko") depends on $dep, which is neither kept nor built in" >&2
+                    exit 1
+                fi
+            done
+        done < <(find "$kernel_dir" -name "*.ko")
+        for name in $MICROVM_EXTRA_MODULES; do
+            if ! grep -qx "${name//-/_}" <<< "$kept_names" && ! module_builtin "$kver_dir" "$name"; then
+                echo "ERROR: required module $name missing from the stripped module tree" >&2
+                exit 1
+            fi
+        done
+        echo "$TAG extra module closure: $(echo "$extra_names" | tr '\n' ' ')"
+        echo "$TAG modules kept: $(echo "$kept_names" | tr '\n' ' ')"
+
+        # Rebuild module metadata with the image's own kmod, the binary the guest's
+        # modprobe runs, so no host depmod is needed. Stock indexes still name .ko.gz
+        # files, so they are removed first and a failed rebuild cannot ship them.
+        for f in dep dep.bin alias alias.bin softdep weakdep symbols symbols.bin \
+            builtin.bin builtin.alias.bin devname; do
+            rm -f "$kver_dir/modules.$f"
+        done
+        if ! image_kmod depmod -b "$CHROOT_DIR" "$kver"; then
+            echo "ERROR: image depmod failed for $kver" >&2
+            exit 1
+        fi
+        if [ ! -s "$kver_dir/modules.dep" ] || [ ! -s "$kver_dir/modules.dep.bin" ]; then
+            echo "ERROR: ${kver_dir}modules.dep or modules.dep.bin missing or empty after depmod" >&2
+            exit 1
+        fi
+        # Resolution gate: the image's modprobe must resolve every required module
+        # to plain .ko files present in the stripped tree.
+        for name in $MICROVM_EXTRA_MODULES $MICROVM_GATE_MODULES; do
+            module_builtin "$kver_dir" "$name" && continue
+            if ! plan=$(image_kmod modprobe -d "$CHROOT_DIR" -S "$kver" --show-depends "$name"); then
+                echo "ERROR: image modprobe cannot resolve $name for $kver" >&2
+                exit 1
+            fi
+            while read -r verb path _; do
+                [ "$verb" = "insmod" ] || continue
+                case "$path" in
+                    "$CHROOT_DIR/lib/modules/$kver/kernel/"*.ko) [ -f "$path" ] && continue ;;
+                esac
+                echo "ERROR: modprobe plan for $name names $path, not a kept plain .ko" >&2
+                exit 1
+            done <<< "$plan"
+            echo "$TAG modprobe $name: $(awk '{ n = split($2, a, "/"); printf "%s ", a[n] }' <<< "$plan")"
+        done
+    done
+}
+
+# microvm_strip_rootfs drops package-manager artifacts and debug symbols.
+microvm_strip_rootfs() {
+    # --- Strip package-manager artifacts (not needed at runtime) ---
+    echo "$TAG stripping package manager artifacts..."
+    rm -rf \
+        "$CHROOT_DIR/lib/apk" \
+        "$CHROOT_DIR/var/cache/apk" \
+        "$CHROOT_DIR/etc/apk" \
+        "$CHROOT_DIR/usr/share/man" \
+        "$CHROOT_DIR/usr/share/doc" \
+        "$CHROOT_DIR/usr/share/apk" \
+        "$CHROOT_DIR/usr/include" \
+        "$CHROOT_DIR/usr/lib/pkgconfig"
+
+    # --- Strip debug symbols from binaries ---
+    echo "$TAG stripping debug symbols from binaries..."
+    find "$CHROOT_DIR/usr/sbin" "$CHROOT_DIR/usr/bin" \
+         "$CHROOT_DIR/sbin" "$CHROOT_DIR/bin" \
+        -type f 2>/dev/null | while read -r bin; do
+        strip --strip-all "$bin" 2>/dev/null || true
+    done
+}
+
+# microvm_stage_kernel copies vmlinuz to $STAGE_DIR, then drops /boot from the chroot.
+microvm_stage_kernel() {
+    # --- Copy vmlinuz out before stripping /boot from chroot ---
+    VMLINUZ_OUT="$STAGE_DIR/vmlinuz"
+    echo "$TAG copying kernel: $VMLINUZ_OUT"
+    cp "$KERNEL_IMG" "$VMLINUZ_OUT"
+
+    # Drop /boot from the chroot — vmlinuz, Alpine's stock initramfs-virt, System.map
+    # and kernel config all live here (~25 MiB). None are needed at runtime: the
+    # guest already has the kernel loaded by QEMU's -kernel, and Alpine's initramfs
+    # is unused because we replace it with our own /init. Leaving /boot in the cpio
+    # nearly doubles uncompressed initramfs size and forces a 512 MiB guest just to
+    # survive decompression.
+    rm -rf "${CHROOT_DIR:?}/boot"
+}
+
+microvm_pack_initramfs() {
+    # --- Build initramfs ---
+    INITRAMFS_OUT="$STAGE_DIR/initramfs.cpio.gz"
+    echo "$TAG building initramfs: $INITRAMFS_OUT"
+    # Create the static device nodes and pack the archive inside a single fakeroot
+    # session: mknod/cpio see faked device entries and write real char-device
+    # records into the cpio, while the host filesystem never gets an actual node.
+    fakeroot sh -c '
+        cd "$1" || exit 1
+        mknod -m 600 dev/console c 5 1
+        mknod -m 666 dev/null    c 1 3
+        find . | cpio --quiet -o -H newc | gzip -9 > "$2"
+    ' _ "$CHROOT_DIR" "$INITRAMFS_OUT"
+}

scripts/setup.sh

@@ -732,7 +732,8 @@ table inet spinifex_filter {
         # inventory and running instances to anyone who asks. 8660 is
         # predastore's unauthenticated /healthz and /readyz, which name the
         # peers a gate cannot reach, so it belongs here for the same reason.
-        ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
+        # 5001 is the admin network test, listening only while one runs.
+        ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5001, 5300, 6641, 6642, 6643, 6644, 8660 } accept
         ip saddr $spinifex_peers udp dport { 5300, 6660, 7660 } accept
 
         # Encap plane, peer-scoped: Geneve, IKE, NAT-T and ESP. Geneve is a
@@ -1309,6 +1310,7 @@ qemu-utils gdisk ovmf qemu-efi-aarch64 less
 libvirt-daemon-system libvirt-clients
 pciutils
 jq curl iproute2 iputils-arping ethtool netcat-openbsd wget unzip xz-utils file
+dmidecode iperf
 ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base
 chrony nftables"
 
@@ -1318,7 +1320,7 @@ chrony nftables"
 OL_RUNTIME_PACKAGES="nbdkit nbdkit-basic-filters
 qemu-kvm qemu-img edk2-ovmf seabios-bin less
 pciutils
-jq curl iproute iputils ethtool nmap-ncat wget unzip xz file
+jq curl iproute iputils ethtool nmap-ncat wget unzip xz file dmidecode
 ovn25.03-central ovn25.03-host openvswitch3.5 openvswitch3.5-ipsec libreswan dhcpcd
 chrony nftables iptables-nft systemd-resolved"
 

spinifex/agents/ecs/agent/credendpoint.go

@@ -47,6 +47,13 @@ type credResponse struct {
 
 // credEntry caches one task's assumed-role credentials alongside the role it was
 // assumed from.
+// credRole is the task role a credID serves and the task it belongs to; the
+// task ID names the STS session, since the credID is a secret.
+type credRole struct {
+	roleARN string
+	taskID  string
+}
+
 type credEntry struct {
 	roleARN string
 	creds   stsauth.Credentials
@@ -71,7 +78,7 @@ type credEndpoint struct {
 	assume func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error)
 
 	mu         sync.Mutex
-	roles      map[string]string    // credID -> taskRoleARN
+	roles      map[string]credRole  // credID -> task role
 	cache      map[string]credEntry // credID -> cached creds
 	httpClient *http.Client
 
@@ -98,7 +105,7 @@ func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayUR
 		port:       port,
 		proxyPort:  defaultCredProxyPort,
 		run:        run,
-		roles:      map[string]string{},
+		roles:      map[string]credRole{},
 		cache:      map[string]credEntry{},
 	}
 	c.assume = c.assumeOverGateway
@@ -107,12 +114,12 @@ func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayUR
 
 // Register associates a credID with the task role it serves; Deregister removes
 // it and drops any cached credentials. Both are no-ops for an empty credID.
-func (c *credEndpoint) Register(credID, roleARN string) {
+func (c *credEndpoint) Register(credID, taskID, roleARN string) {
 	if credID == "" || roleARN == "" {
 		return
 	}
 	c.mu.Lock()
-	c.roles[credID] = roleARN
+	c.roles[credID] = credRole{roleARN: roleARN, taskID: taskID}
 	c.mu.Unlock()
 }
 
@@ -142,25 +149,25 @@ func (c *credEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "not found", http.StatusNotFound)
 		return
 	}
-	creds, roleARN, err := c.fetch(r.Context(), credID)
+	creds, role, err := c.fetch(r.Context(), credID)
 	if err != nil {
 		if errors.Is(err, errUnknownCredID) {
 			http.Error(w, "not found", http.StatusNotFound)
 			return
 		}
-		slog.Error("ecs-agent: assume task role failed", "credId", credID, "err", err)
+		slog.Error("ecs-agent: assume task role failed", "task", role.taskID, "role", role.roleARN, "err", err)
 		http.Error(w, "credentials unavailable", http.StatusBadGateway)
 		return
 	}
 	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(credResponse{
-		RoleArn:         roleARN,
+		RoleArn:         role.roleARN,
 		AccessKeyID:     creds.AccessKeyID,
 		SecretAccessKey: creds.SecretAccessKey,
 		Token:           creds.SessionToken,
 		Expiration:      creds.Expiration.UTC().Format(time.RFC3339),
 	}); err != nil {
-		slog.Warn("ecs-agent: encode credential response", "credId", credID, "err", err)
+		slog.Warn("ecs-agent: encode credential response", "task", role.taskID, "err", err)
 	}
 }
 
@@ -169,28 +176,28 @@ var errUnknownCredID = fmt.Errorf("unknown credId")
 
 // fetch returns valid credentials for credID, reusing the cache until within the
 // refresh margin and otherwise assuming the task role over the gateway.
-func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, string, error) {
+func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, credRole, error) {
 	c.mu.Lock()
-	roleARN, known := c.roles[credID]
+	role, known := c.roles[credID]
 	if !known {
 		c.mu.Unlock()
-		return stsauth.Credentials{}, "", errUnknownCredID
+		return stsauth.Credentials{}, credRole{}, errUnknownCredID
 	}
-	if e, ok := c.cache[credID]; ok && e.roleARN == roleARN && credValid(e.creds, credRefreshMargin) {
+	if e, ok := c.cache[credID]; ok && e.roleARN == role.roleARN && credValid(e.creds, credRefreshMargin) {
 		c.mu.Unlock()
-		return e.creds, roleARN, nil
+		return e.creds, role, nil
 	}
 	c.mu.Unlock()
 
-	creds, err := c.assume(ctx, roleARN, sessionName(credID))
+	creds, err := c.assume(ctx, role.roleARN, sessionName(role.taskID))
 	if err != nil {
-		return stsauth.Credentials{}, "", err
+		return stsauth.Credentials{}, role, err
 	}
 
 	c.mu.Lock()
-	c.cache[credID] = credEntry{roleARN: roleARN, creds: creds}
+	c.cache[credID] = credEntry{roleARN: role.roleARN, creds: creds}
 	c.mu.Unlock()
-	return creds, roleARN, nil
+	return creds, role, nil
 }
 
 // AssumeProvider returns a CredentialsProvider that mints credentials by assuming
@@ -390,10 +397,10 @@ func credValid(c stsauth.Credentials, margin time.Duration) bool {
 	return time.Now().Add(margin).Before(c.Expiration)
 }
 
-// sessionName derives an AssumeRole RoleSessionName from credID, satisfying the
+// sessionName derives an AssumeRole RoleSessionName from the task ID, satisfying the
 // STS [\w+=,.@-]{2,64} constraint.
-func sessionName(credID string) string {
-	name := "ecs-" + credID
+func sessionName(taskID string) string {
+	name := "ecs-" + taskID
 	if len(name) > 64 {
 		name = name[:64]
 	}

spinifex/agents/ecs/agent/credendpoint_test.go

@@ -29,7 +29,7 @@ func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expiration: exp,
 	})
-	c.Register("cred-1", "arn:aws:iam::111122223333:role/task")
+	c.Register("cred-1", "task-1", "arn:aws:iam::111122223333:role/task")
 
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -51,6 +51,26 @@ func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 	}
 }
 
+// The STS session name lands in the assumed-role ARN and in STS logs, so it is
+// built from the public task ID and never from the secret credID.
+func TestCredEndpoint_SessionNameIsTaskIDNotCredID(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{})
+	var gotSession string
+	c.assume = func(ctx context.Context, roleARN, session string) (stsauth.Credentials, error) {
+		gotSession = session
+		return stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"}, nil
+	}
+	c.Register("secret-cred", "task-1", "arn:role")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/secret-cred", nil))
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rr.Code)
+	}
+	if gotSession != "ecs-task-1" {
+		t.Errorf("session name = %q, want ecs-task-1", gotSession)
+	}
+}
+
 func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
 	c, calls := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
 	rr := httptest.NewRecorder()
@@ -65,7 +85,7 @@ func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
 
 func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/v2/credentials/cred-1", nil))
 	if rr.Code != http.StatusMethodNotAllowed {
@@ -77,7 +97,7 @@ func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
 	c, calls := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
 	})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	for hit := range 3 {
 		rr := httptest.NewRecorder()
 		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -95,7 +115,7 @@ func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 	c, calls := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Minute),
 	})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	for range 2 {
 		rr := httptest.NewRecorder()
 		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -107,7 +127,7 @@ func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 
 func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	c.Deregister("cred-1")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -121,7 +141,7 @@ func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
 	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
 		return stsauth.Credentials{}, context.DeadlineExceeded
 	}
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
 	if rr.Code != http.StatusBadGateway {
@@ -133,7 +153,7 @@ func TestCredEndpoint_StartServesOverLoopback(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
 	})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	c.port = 0 // bind an ephemeral port instead of privileged 80
 	if err := c.Start(); err != nil {
 		t.Fatalf("start: %v", err)

spinifex/agents/ecs/agent/credinject_test.go

@@ -1,8 +1,12 @@
 package agent
 
 import (
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
+	"github.com/mulgadc/spinifex/internal/stsauth"
+
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 )
 
@@ -13,7 +17,7 @@ func TestTaskCredID(t *testing.T) {
 		want string
 	}{
 		{"no role", ecsv1.Assign{TaskID: "t1"}, ""},
-		{"role, no credID -> taskID", ecsv1.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, "t1"},
+		{"role, no credID -> none", ecsv1.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, ""},
 		{"role + credID", ecsv1.Assign{TaskID: "t1", CredID: "c9", TaskRoleARN: "arn:role"}, "c9"},
 	}
 	for _, tc := range cases {
@@ -23,6 +27,37 @@ func TestTaskCredID(t *testing.T) {
 	}
 }
 
+// The task ID is public (it is the task ARN suffix), so naming another task's ID
+// at the credential endpoint must not yield that task's role.
+func TestRegisterTaskCreds_TaskIDDoesNotServeCredentials(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"})
+	a := Agent{cred: c}
+	get := func(id string) int {
+		rr := httptest.NewRecorder()
+		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, credRelativeURI(id), nil))
+		return rr.Code
+	}
+
+	legacy := ecsv1.Assign{TaskID: "victim-task", TaskRoleARN: "arn:aws:iam::000000000010:role/high"}
+	if got := a.registerTaskCreds(&legacy); got != "" {
+		t.Errorf("assign without CredID registered credID %q", got)
+	}
+	if code := get("victim-task"); code != http.StatusNotFound {
+		t.Errorf("GET by task ID of CredID-less assign = %d, want 404", code)
+	}
+
+	minted := ecsv1.Assign{TaskID: "victim-2", CredID: "secret-cred", TaskRoleARN: "arn:aws:iam::000000000010:role/high"}
+	if got := a.registerTaskCreds(&minted); got != "secret-cred" {
+		t.Fatalf("registerTaskCreds = %q, want secret-cred", got)
+	}
+	if code := get("victim-2"); code != http.StatusNotFound {
+		t.Errorf("GET by task ID = %d, want 404", code)
+	}
+	if code := get("secret-cred"); code != http.StatusOK {
+		t.Errorf("GET by CredID = %d, want 200", code)
+	}
+}
+
 func TestWithCredEnv(t *testing.T) {
 	base := map[string]string{"FOO": "bar"}
 

spinifex/agents/ecs/agent/reconcile_test.go

@@ -63,7 +63,7 @@ func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
 	if !adopted["t-001"] {
 		t.Fatalf("t-001 not adopted; got %v", adopted)
 	}
-	if got := a.cred.roles["t-001"]; got != testTaskRole {
+	if got := a.cred.roles["t-001"].roleARN; got != testTaskRole {
 		t.Errorf("cred not re-registered: roles[t-001] = %q, want %q", got, testTaskRole)
 	}
 	if n := countStatus(cp.taskStates(), "t-001", ecsv1.TaskStatusRunning); n != 1 {

spinifex/agents/ecs/agent/taskrun.go

@@ -306,24 +306,24 @@ func (a *Agent) pullResolver(as *ecsv1.Assign) ctrruntime.Resolver {
 // credential endpoint and returns the credID (empty when the task has no role).
 func (a *Agent) registerTaskCreds(as *ecsv1.Assign) string {
 	credID := taskCredID(as)
+	if credID == "" && as.TaskRoleARN != "" {
+		slog.Warn("ecs-agent: task has a role but no credential ID, starting it without credentials", "task", as.TaskID, "role", as.TaskRoleARN)
+	}
 	if credID == "" || a.cred == nil {
 		return credID
 	}
-	a.cred.Register(credID, as.TaskRoleARN)
+	a.cred.Register(credID, as.TaskID, as.TaskRoleARN)
 	return credID
 }
 
 // taskCredID is the credential ID a task's containers fetch credentials under:
-// the scheduler-assigned CredID, falling back to the taskID. Empty when the task
-// carries no IAM role.
+// the scheduler-minted secret CredID. Empty when the task carries no IAM role or
+// no CredID; the public task ID is never used, since any API caller can learn it.
 func taskCredID(as *ecsv1.Assign) string {
 	if as.TaskRoleARN == "" {
 		return ""
 	}
-	if as.CredID != "" {
-		return as.CredID
-	}
-	return as.TaskID
+	return as.CredID
 }
 
 // withCredEnv returns env with AWS_CONTAINER_CREDENTIALS_RELATIVE_URI set for

spinifex/agents/lambda/agent/agent.go

@@ -0,0 +1,91 @@
+// Package agent runs the guest loop: take a request from the host link, hand
+// it to the function runtime, and report the outcome back.
+package agent
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+)
+
+const (
+	errTypeTimedOut = "Sandbox.Timedout"
+	errTypeUnknown  = "Runtime.Unknown"
+)
+
+// ErrInvocationTimedOut stops the agent after it reports a timed-out
+// invocation, because the runtime may still be running it.
+var ErrInvocationTimedOut = errors.New("invocation timed out; environment must not serve another request")
+
+// Run serves requests from link until the host asks for shutdown (nil), the
+// runtime reports an init error, an invocation times out, ctx ends, or the
+// link fails. GrantID and AttemptID are echoed to the host unchanged.
+func Run(ctx context.Context, link hostlink.Link, srv *runtimeapi.Server) error {
+	ctx, cancel := context.WithCancelCause(ctx)
+	defer cancel(nil)
+	go func() {
+		select {
+		case <-srv.InitFailed():
+			ie, _ := srv.InitError()
+			cancel(fmt.Errorf("runtime init error %q: %s", ie.Type, ie.Body))
+		case <-ctx.Done():
+		}
+	}()
+
+	for {
+		req, err := link.Next(ctx)
+		if errors.Is(err, hostlink.ErrShutdown) {
+			return nil
+		}
+		if err != nil {
+			return cause(ctx, err)
+		}
+
+		res, err := srv.Invoke(ctx, runtimeapi.Invocation{
+			RequestID:   req.RequestID,
+			FunctionARN: req.FunctionARN,
+			Deadline:    time.UnixMilli(req.DeadlineMs),
+			TraceID:     req.TraceID,
+			Payload:     req.Payload,
+		})
+		timedOut := errors.Is(err, runtimeapi.ErrDeadlineExceeded)
+		if err != nil {
+			if ctx.Err() != nil {
+				return cause(ctx, err)
+			}
+			res = failure(err)
+		}
+		resp := hostlink.Response{
+			GrantID:       req.GrantID,
+			AttemptID:     req.AttemptID,
+			RequestID:     req.RequestID,
+			Payload:       res.Payload,
+			FunctionError: res.FunctionError,
+		}
+		if err := link.Complete(ctx, resp); err != nil {
+			return cause(ctx, err)
+		}
+		if timedOut {
+			return ErrInvocationTimedOut
+		}
+	}
+}
+
+func cause(ctx context.Context, err error) error {
+	if ctx.Err() != nil {
+		return context.Cause(ctx)
+	}
+	return err
+}
+
+func failure(err error) runtimeapi.Result {
+	errType := errTypeUnknown
+	if errors.Is(err, runtimeapi.ErrDeadlineExceeded) {
+		errType = errTypeTimedOut
+	}
+	return runtimeapi.Result{Payload: runtimeapi.ErrorBody(err.Error(), errType), FunctionError: errType}
+}

spinifex/agents/lambda/agent/agent_test.go

@@ -0,0 +1,142 @@
+package agent
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+)
+
+const api = "/2018-06-01/runtime/"
+
+// fakeLink feeds scripted requests and records completions.
+type fakeLink struct {
+	reqs []hostlink.Request
+	got  chan hostlink.Response
+}
+
+var _ hostlink.Link = (*fakeLink)(nil)
+
+func (l *fakeLink) Next(ctx context.Context) (hostlink.Request, error) {
+	if len(l.reqs) == 0 {
+		return hostlink.Request{}, hostlink.ErrShutdown
+	}
+	r := l.reqs[0]
+	l.reqs = l.reqs[1:]
+	return r, nil
+}
+
+func (l *fakeLink) Complete(ctx context.Context, r hostlink.Response) error {
+	l.got <- r
+	return nil
+}
+
+// runtime polls next and answers using reply.
+func runtime(t *testing.T, base string, reply func(id string, body []byte) (path string, out string)) {
+	t.Helper()
+	go func() {
+		for {
+			resp, err := http.Get(base + api + "invocation/next")
+			if err != nil {
+				return
+			}
+			body, _ := io.ReadAll(resp.Body)
+			resp.Body.Close()
+			id := resp.Header.Get("Lambda-Runtime-Aws-Request-Id")
+			path, out := reply(id, body)
+			r, err := http.Post(base+api+"invocation/"+id+path, "application/json", strings.NewReader(out))
+			if err != nil {
+				return
+			}
+			r.Body.Close()
+		}
+	}()
+}
+
+func deadline(d time.Duration) int64 { return time.Now().Add(d).UnixMilli() }
+
+func TestRunServesRequestsAndEchoesIDs(t *testing.T) {
+	srv := runtimeapi.New()
+	ts := httptest.NewServer(srv.Handler())
+	defer ts.Close()
+	runtime(t, ts.URL, func(id string, body []byte) (string, string) {
+		if string(body) == "bad" {
+			return "/error", `{"errorMessage":"no"}`
+		}
+		return "/response", "echo:" + string(body)
+	})
+
+	link := &fakeLink{
+		got: make(chan hostlink.Response, 4),
+		reqs: []hostlink.Request{
+			{GrantID: "g1", AttemptID: "a1", RequestID: "r1", FunctionARN: "arn", DeadlineMs: deadline(2 * time.Second), Payload: []byte("one")},
+			{GrantID: "g2", AttemptID: "a2", RequestID: "r2", FunctionARN: "arn", DeadlineMs: deadline(2 * time.Second), Payload: []byte("bad")},
+		},
+	}
+	require.NoError(t, Run(context.Background(), link, srv))
+
+	r1, r2 := <-link.got, <-link.got
+	assert.Equal(t, hostlink.Response{GrantID: "g1", AttemptID: "a1", RequestID: "r1", Payload: []byte("echo:one")}, r1)
+	assert.Equal(t, "g2", r2.GrantID)
+	assert.Equal(t, "a2", r2.AttemptID)
+	assert.Equal(t, "Unhandled", r2.FunctionError)
+}
+
+func TestRunReportsTimeoutAndStops(t *testing.T) {
+	srv := runtimeapi.New() // no runtime polling: the invocation expires
+	link := &fakeLink{
+		got: make(chan hostlink.Response, 2),
+		reqs: []hostlink.Request{
+			{GrantID: "g", AttemptID: "a", RequestID: "r", DeadlineMs: deadline(30 * time.Millisecond)},
+			{GrantID: "g2", AttemptID: "a2", RequestID: "r2", DeadlineMs: deadline(2 * time.Second)},
+		},
+	}
+	require.ErrorIs(t, Run(context.Background(), link, srv), ErrInvocationTimedOut)
+	r := <-link.got
+	assert.Equal(t, "Sandbox.Timedout", r.FunctionError)
+	assert.Equal(t, "g", r.GrantID)
+	assert.Len(t, link.reqs, 1, "the next request must not be taken")
+	assert.Empty(t, link.got, "nothing else is reported")
+}
+
+// blockLink blocks in Next until ctx ends.
+type blockLink struct{ fakeLink }
+
+func (blockLink) Next(ctx context.Context) (hostlink.Request, error) {
+	<-ctx.Done()
+	return hostlink.Request{}, ctx.Err()
+}
+
+func TestRunStopsOnInitError(t *testing.T) {
+	srv := runtimeapi.New()
+	ts := httptest.NewServer(srv.Handler())
+	defer ts.Close()
+	r, err := http.Post(ts.URL+api+"init/error", "application/json", strings.NewReader(`{"errorMessage":"x"}`))
+	require.NoError(t, err)
+	r.Body.Close()
+
+	err = Run(context.Background(), &blockLink{}, srv)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "init error")
+}
+
+type errLink struct{ fakeLink }
+
+func (errLink) Next(context.Context) (hostlink.Request, error) {
+	return hostlink.Request{}, errors.New("link down")
+}
+
+func TestRunReturnsLinkError(t *testing.T) {
+	err := Run(context.Background(), &errLink{}, runtimeapi.New())
+	require.EqualError(t, err, "link down")
+}

spinifex/agents/lambda/agent/bootstrap.go

@@ -0,0 +1,54 @@
+//go:build linux
+
+package agent
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os/exec"
+	goruntime "runtime"
+	"syscall"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// sandboxID is the uid and gid the function's bootstrap runs as.
+const sandboxID = 993
+
+// StartBootstrap starts the function's bootstrap at path as the sandbox uid and
+// gid with no supplementary groups, with no_new_privs, and with exactly env.
+// The caller waits on the returned command.
+func StartBootstrap(ctx context.Context, path string, env []string, stdout, stderr io.Writer) (*exec.Cmd, error) {
+	cred := &syscall.Credential{Uid: sandboxID, Gid: sandboxID, Groups: []uint32{}}
+	return startBootstrap(ctx, path, env, stdout, stderr, cred)
+}
+
+func startBootstrap(ctx context.Context, path string, env []string, stdout, stderr io.Writer, cred *syscall.Credential) (*exec.Cmd, error) {
+	cmd := exec.CommandContext(ctx, path)
+	// A nil Env inherits the agent's environment; the bootstrap gets only env.
+	cmd.Env = append([]string{}, env...)
+	cmd.Stdout, cmd.Stderr = stdout, stderr
+	if cred != nil {
+		cmd.SysProcAttr = &syscall.SysProcAttr{Credential: cred}
+	}
+	cmd.Cancel = func() error { return cmd.Process.Signal(syscall.SIGTERM) }
+	cmd.WaitDelay = 5 * time.Second
+
+	started := make(chan error, 1)
+	go func() {
+		// no_new_privs is per thread and the child inherits it from this one. The
+		// thread is never unlocked, so Go ends it when this goroutine returns.
+		goruntime.LockOSThread()
+		if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {
+			started <- fmt.Errorf("set no_new_privs: %w", err)
+			return
+		}
+		started <- cmd.Start()
+	}()
+	if err := <-started; err != nil {
+		return nil, err
+	}
+	return cmd, nil
+}

spinifex/agents/lambda/agent/bootstrap_test.go

@@ -0,0 +1,171 @@
+//go:build linux
+
+package agent
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	goruntime "runtime"
+	"strings"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// statusProbe writes an executable that prints its own credential lines.
+func statusProbe(t *testing.T) string {
+	t.Helper()
+	// The sandbox uid must be able to reach the probe through the test's dirs.
+	dir := t.TempDir()
+	require.NoError(t, os.Chmod(filepath.Dir(dir), 0o755))
+	require.NoError(t, os.Chmod(dir, 0o755))
+	path := filepath.Join(dir, "bootstrap")
+	script := "#!/bin/sh\ngrep -E '^(Uid|Gid|Groups|NoNewPrivs):' /proc/self/status\n"
+	require.NoError(t, os.WriteFile(path, []byte(script), 0o755))
+	return path
+}
+
+func statusLines(t *testing.T, out []byte) map[string]string {
+	t.Helper()
+	lines := map[string]string{}
+	for l := range strings.SplitSeq(strings.TrimSpace(string(out)), "\n") {
+		k, v, ok := strings.Cut(l, ":")
+		require.True(t, ok, "status line %q", l)
+		lines[k] = strings.Join(strings.Fields(v), " ")
+	}
+	return lines
+}
+
+// run starts a bootstrap with the given credential and returns its stdout.
+func run(t *testing.T, path string, env []string, sandbox bool) []byte {
+	t.Helper()
+	var out bytes.Buffer
+	start := startBootstrap
+	if sandbox {
+		start = func(ctx context.Context, p string, e []string, o, s io.Writer, _ *syscall.Credential) (*exec.Cmd, error) {
+			return StartBootstrap(ctx, p, e, o, s)
+		}
+	}
+	cmd, err := start(context.Background(), path, env, &out, os.Stderr, nil)
+	require.NoError(t, err)
+	require.NoError(t, cmd.Wait())
+	return out.Bytes()
+}
+
+// threadNoNewPrivs reads no_new_privs for the calling OS thread.
+func threadNoNewPrivs(t *testing.T) string {
+	t.Helper()
+	b, err := os.ReadFile("/proc/thread-self/status")
+	require.NoError(t, err)
+	return statusLines(t, grepLines(b, "NoNewPrivs:"))["NoNewPrivs"]
+}
+
+func grepLines(b []byte, prefix string) []byte {
+	var keep []string
+	for l := range strings.SplitSeq(string(b), "\n") {
+		if strings.HasPrefix(l, prefix) {
+			keep = append(keep, l)
+		}
+	}
+	return []byte(strings.Join(keep, "\n"))
+}
+
+func TestStartBootstrapSetsNoNewPrivsWithoutChangingTheCaller(t *testing.T) {
+	goruntime.LockOSThread()
+	defer goruntime.UnlockOSThread()
+	if threadNoNewPrivs(t) != "0" {
+		t.Skip("test thread already has no_new_privs; the child would inherit it")
+	}
+
+	out := run(t, statusProbe(t), nil, false)
+	assert.Equal(t, "1", statusLines(t, out)["NoNewPrivs"])
+	assert.Equal(t, "0", threadNoNewPrivs(t), "the calling thread must keep no_new_privs unset")
+}
+
+func TestStartBootstrapDropsToSandboxIdentity(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip("dropping to the sandbox uid needs root")
+	}
+	lines := statusLines(t, run(t, statusProbe(t), nil, true))
+	assert.Equal(t, "993 993 993 993", lines["Uid"])
+	assert.Equal(t, "993 993 993 993", lines["Gid"])
+	assert.Empty(t, lines["Groups"])
+	assert.Equal(t, "1", lines["NoNewPrivs"])
+}
+
+// A tenant LD_PRELOAD constructor runs before the bootstrap's main; it must
+// already see no_new_privs, so no loader runs before the flag is set.
+func TestStartBootstrapPreloadConstructorSeesNoNewPrivs(t *testing.T) {
+	goruntime.LockOSThread()
+	defer goruntime.UnlockOSThread()
+	if threadNoNewPrivs(t) != "0" {
+		t.Skip("test thread already has no_new_privs; the check would pass vacuously")
+	}
+	cc, err := exec.LookPath("cc")
+	if err != nil {
+		t.Skip("no C compiler for the preload library")
+	}
+	bootstrap, err := exec.LookPath("true")
+	require.NoError(t, err)
+	dir := t.TempDir()
+	src := filepath.Join(dir, "preload.c")
+	lib := filepath.Join(dir, "preload.so")
+	require.NoError(t, os.WriteFile(src, []byte(preloadSource), 0o644))
+	build := exec.Command(cc, "-shared", "-fPIC", "-o", lib, src)
+	build.Stderr = os.Stderr
+	require.NoError(t, build.Run())
+
+	out := run(t, bootstrap, []string{"LD_PRELOAD=" + lib}, false)
+	assert.Equal(t, "preload NoNewPrivs: 1", strings.Join(strings.Fields(string(out)), " "))
+}
+
+const preloadSource = `#include <stdio.h>
+#include <string.h>
+
+__attribute__((constructor)) static void report(void) {
+	char line[256];
+	FILE *f = fopen("/proc/self/status", "r");
+	if (!f)
+		return;
+	while (fgets(line, sizeof line, f))
+		if (strncmp(line, "NoNewPrivs:", 11) == 0)
+			printf("preload %s", line);
+	fclose(f);
+	fflush(stdout);
+}
+`
+
+func TestStartBootstrapDoesNotInheritAgentEnvironment(t *testing.T) {
+	t.Setenv("ZZ_AGENT_SECRET", "must-not-leak")
+	probe, err := exec.LookPath("env")
+	require.NoError(t, err)
+
+	for name, env := range map[string][]string{"nil": nil, "empty": {}, "given": {"ONLY=this"}} {
+		out := run(t, probe, env, false)
+		// Never print the captured environment: an inherited one may hold secrets.
+		if strings.TrimSuffix(string(out), "\n") != strings.Join(env, "\n") {
+			t.Errorf("%s: bootstrap environment is not exactly the one given", name)
+		}
+	}
+}
+
+// Without root the sandbox credential cannot be applied, so the exported entry
+// point must refuse rather than run the bootstrap as the caller.
+func TestStartBootstrapAsNonRootRefusesInsteadOfKeepingCallerIdentity(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("root can apply the sandbox credential")
+	}
+	_, err := StartBootstrap(context.Background(), statusProbe(t), nil, nil, nil)
+	assert.ErrorIs(t, err, syscall.EPERM)
+}
+
+func TestStartBootstrapMissingExecutableFails(t *testing.T) {
+	_, err := startBootstrap(context.Background(), filepath.Join(t.TempDir(), "bootstrap"), nil, nil, nil, nil)
+	assert.Error(t, err)
+}

spinifex/agents/lambda/hostlink/link.go

@@ -0,0 +1,41 @@
+// Package hostlink is the guest's boundary to host control: an authenticated
+// TCP connection under per-generation mutual TLS. The message protocol over it
+// is not yet chosen; implementations of Link are carriers.
+package hostlink
+
+import (
+	"context"
+	"errors"
+)
+
+// ErrShutdown is returned by Link.Next when the host asks the agent to stop.
+var ErrShutdown = errors.New("hostlink: host requested shutdown")
+
+// Request is one invocation from the host. GrantID and AttemptID are opaque
+// to the agent and are echoed unchanged in the Response.
+type Request struct {
+	GrantID     string `json:"grant_id"`
+	AttemptID   string `json:"attempt_id"`
+	RequestID   string `json:"request_id"`
+	FunctionARN string `json:"function_arn"`
+	TraceID     string `json:"trace_id,omitempty"`
+	DeadlineMs  int64  `json:"deadline_ms"`
+	Payload     []byte `json:"payload"`
+}
+
+// Response is the outcome of a Request. FunctionError is empty on success.
+type Response struct {
+	GrantID       string `json:"grant_id"`
+	AttemptID     string `json:"attempt_id"`
+	RequestID     string `json:"request_id"`
+	Payload       []byte `json:"payload"`
+	FunctionError string `json:"function_error,omitempty"`
+}
+
+// Link carries invocations from the host to the guest and results back.
+type Link interface {
+	// Next blocks for the next request, or returns ErrShutdown.
+	Next(ctx context.Context) (Request, error)
+	// Complete reports the outcome of a request.
+	Complete(ctx context.Context, resp Response) error
+}

spinifex/agents/lambda/hostlink/seal.go

@@ -0,0 +1,42 @@
+package hostlink
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+// FwCfgRoot is the sysfs tree the qemu_fw_cfg module exposes; removing the
+// module removes every entry under it, including the guest key.
+const FwCfgRoot = "/sys/firmware/qemu_fw_cfg"
+
+// UnloadFwCfg removes the qemu_fw_cfg module from the running kernel.
+func UnloadFwCfg() error {
+	if err := unix.DeleteModule("qemu_fw_cfg", 0); err != nil {
+		return fmt.Errorf("hostlink: unload qemu_fw_cfg: %w", err)
+	}
+	return nil
+}
+
+// SealFwCfg makes fw_cfg unreadable for the rest of the boot. It succeeds only
+// once root is gone, so a built-in or busy module fails closed rather than
+// leaving the guest key readable by code started later.
+func SealFwCfg(root string, unload func() error) error {
+	if _, err := os.Stat(root); errors.Is(err, fs.ErrNotExist) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("hostlink: fw_cfg: %w", err)
+	}
+	unloadErr := unload()
+	_, err := os.Stat(root)
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil
+	}
+	if unloadErr == nil {
+		unloadErr = errors.New("unload reported success")
+	}
+	return fmt.Errorf("hostlink: fw_cfg still readable at %s: %w", root, errors.Join(unloadErr, err))
+}

spinifex/agents/lambda/hostlink/seal_test.go

@@ -0,0 +1,43 @@
+package hostlink
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func fwCfgTree(t *testing.T) string {
+	t.Helper()
+	root := filepath.Join(t.TempDir(), "qemu_fw_cfg")
+	require.NoError(t, os.MkdirAll(filepath.Join(root, "by_name"), 0o700))
+	return root
+}
+
+func TestSealFwCfg(t *testing.T) {
+	t.Run("already absent needs no unload", func(t *testing.T) {
+		called := false
+		require.NoError(t, SealFwCfg(filepath.Join(t.TempDir(), "absent"), func() error { called = true; return nil }))
+		assert.False(t, called)
+	})
+	t.Run("unload removes the tree", func(t *testing.T) {
+		root := fwCfgTree(t)
+		require.NoError(t, SealFwCfg(root, func() error { return os.RemoveAll(root) }))
+	})
+	t.Run("unload fails and the tree remains", func(t *testing.T) {
+		builtIn := errors.New("module is built in")
+		err := SealFwCfg(fwCfgTree(t), func() error { return builtIn })
+		assert.ErrorIs(t, err, builtIn)
+		assert.ErrorContains(t, err, "still readable")
+	})
+	t.Run("unload succeeds but the tree remains", func(t *testing.T) {
+		assert.ErrorContains(t, SealFwCfg(fwCfgTree(t), func() error { return nil }), "unload reported success")
+	})
+	t.Run("unload errors after removing the tree", func(t *testing.T) {
+		root := fwCfgTree(t)
+		require.NoError(t, SealFwCfg(root, func() error { _ = os.RemoveAll(root); return errors.New("late") }))
+	})
+}

spinifex/agents/lambda/hostlink/test_stream.go

@@ -0,0 +1,163 @@
+//go:build lambdatestlink
+
+package hostlink
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"sync"
+)
+
+// maxFrame bounds one frame: a 6 MiB payload as base64 plus headroom.
+const maxFrame = (6<<20+2)/3*4 + 64<<10
+
+const (
+	typeInvoke   = "invoke"
+	typeShutdown = "shutdown"
+	typeResult   = "result"
+)
+
+// ErrFrameTooLarge is returned for a frame beyond the size bound.
+var ErrFrameTooLarge = errors.New("hostlink: frame too large")
+
+type frame struct {
+	Type     string    `json:"type"`
+	Request  *Request  `json:"request,omitempty"`
+	Response *Response `json:"response,omitempty"`
+}
+
+// TestStream is a test-only carrier over an io.ReadWriter, framing JSON with
+// a 4-byte big-endian length. It is not the guest protocol.
+type TestStream struct {
+	rw  io.ReadWriter
+	in  chan frame
+	wmu sync.Mutex
+
+	err error // set before in is closed
+}
+
+var _ Link = (*TestStream)(nil)
+
+// NewTestStream starts reading frames from rw.
+func NewTestStream(rw io.ReadWriter) *TestStream {
+	s := &TestStream{rw: rw, in: make(chan frame)}
+	go s.readLoop()
+	return s
+}
+
+func (s *TestStream) readLoop() {
+	defer close(s.in)
+	for {
+		f, err := readFrame(s.rw)
+		if err != nil {
+			s.err = err
+			return
+		}
+		s.in <- f
+	}
+}
+
+func (s *TestStream) recv(ctx context.Context) (frame, error) {
+	select {
+	case f, ok := <-s.in:
+		if !ok {
+			return frame{}, s.err
+		}
+		return f, nil
+	case <-ctx.Done():
+		return frame{}, ctx.Err()
+	}
+}
+
+// Next implements Link.
+func (s *TestStream) Next(ctx context.Context) (Request, error) {
+	f, err := s.recv(ctx)
+	if err != nil {
+		return Request{}, err
+	}
+	switch f.Type {
+	case typeShutdown:
+		return Request{}, ErrShutdown
+	case typeInvoke:
+		if f.Request == nil {
+			return Request{}, errors.New("hostlink: invoke frame without request")
+		}
+		return *f.Request, nil
+	default:
+		return Request{}, fmt.Errorf("hostlink: unexpected frame type %q", f.Type)
+	}
+}
+
+// Complete implements Link.
+func (s *TestStream) Complete(ctx context.Context, resp Response) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	return s.write(frame{Type: typeResult, Response: &resp})
+}
+
+// HostSend writes an invoke frame, as the host side of the carrier.
+func (s *TestStream) HostSend(req Request) error {
+	return s.write(frame{Type: typeInvoke, Request: &req})
+}
+
+// HostShutdown writes a shutdown frame, as the host side of the carrier.
+func (s *TestStream) HostShutdown() error {
+	return s.write(frame{Type: typeShutdown})
+}
+
+// HostResult reads a result frame, as the host side of the carrier.
+func (s *TestStream) HostResult(ctx context.Context) (Response, error) {
+	f, err := s.recv(ctx)
+	if err != nil {
+		return Response{}, err
+	}
+	if f.Type != typeResult || f.Response == nil {
+		return Response{}, fmt.Errorf("hostlink: unexpected frame type %q", f.Type)
+	}
+	return *f.Response, nil
+}
+
+func (s *TestStream) write(f frame) error {
+	b, err := json.Marshal(f)
+	if err != nil {
+		return err
+	}
+	if len(b) > maxFrame {
+		return ErrFrameTooLarge
+	}
+	buf := make([]byte, 4+len(b))
+	binary.BigEndian.PutUint32(buf, uint32(len(b)))
+	copy(buf[4:], b)
+	s.wmu.Lock()
+	defer s.wmu.Unlock()
+	_, err = s.rw.Write(buf)
+	return err
+}
+
+func readFrame(r io.Reader) (frame, error) {
+	var hdr [4]byte
+	if _, err := io.ReadFull(r, hdr[:]); err != nil {
+		return frame{}, err
+	}
+	n := binary.BigEndian.Uint32(hdr[:])
+	if n > maxFrame {
+		return frame{}, ErrFrameTooLarge
+	}
+	body := make([]byte, n)
+	if _, err := io.ReadFull(r, body); err != nil {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		return frame{}, err
+	}
+	var f frame
+	if err := json.Unmarshal(body, &f); err != nil {
+		return frame{}, fmt.Errorf("hostlink: bad frame: %w", err)
+	}
+	return f, nil
+}

spinifex/agents/lambda/hostlink/test_stream_test.go

@@ -0,0 +1,73 @@
+//go:build lambdatestlink
+
+package hostlink
+
+import (
+	"context"
+	"encoding/binary"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// pair returns the guest carrier and the raw host end of a pipe.
+func pair(t *testing.T) (*TestStream, net.Conn) {
+	t.Helper()
+	g, h := net.Pipe()
+	t.Cleanup(func() { g.Close(); h.Close() })
+	return NewTestStream(g), h
+}
+
+func TestRoundTrip(t *testing.T) {
+	guest, host := pair(t)
+	hostEnd := NewTestStream(host)
+	req := Request{GrantID: "g", AttemptID: "a", RequestID: "r", FunctionARN: "arn", TraceID: "t", DeadlineMs: 42, Payload: []byte{0, 1, 2, 255}}
+	go func() { _ = hostEnd.HostSend(req) }()
+
+	got, err := guest.Next(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, req, got)
+
+	resp := Response{GrantID: "g", AttemptID: "a", RequestID: "r", Payload: []byte("out"), FunctionError: "Unhandled"}
+	go func() { _ = guest.Complete(context.Background(), resp) }()
+	gotResp, err := hostEnd.HostResult(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, resp, gotResp)
+}
+
+func TestOversizeFrameRefused(t *testing.T) {
+	guest, host := pair(t)
+	go func() {
+		var hdr [4]byte
+		binary.BigEndian.PutUint32(hdr[:], maxFrame+1)
+		_, _ = host.Write(hdr[:])
+	}()
+	_, err := guest.Next(context.Background())
+	require.ErrorIs(t, err, ErrFrameTooLarge)
+
+	err = guest.Complete(context.Background(), Response{Payload: make([]byte, maxFrame)})
+	require.ErrorIs(t, err, ErrFrameTooLarge)
+}
+
+func TestTruncatedFrameErrors(t *testing.T) {
+	guest, host := pair(t)
+	go func() {
+		var hdr [4]byte
+		binary.BigEndian.PutUint32(hdr[:], 100)
+		_, _ = host.Write(append(hdr[:], []byte(`{"type"`)...))
+		host.Close()
+	}()
+	_, err := guest.Next(context.Background())
+	require.Error(t, err)
+	assert.NotErrorIs(t, err, ErrShutdown)
+}
+
+func TestShutdownSurfaced(t *testing.T) {
+	guest, host := pair(t)
+	hostEnd := NewTestStream(host)
+	go func() { _ = hostEnd.HostShutdown() }()
+	_, err := guest.Next(context.Background())
+	require.ErrorIs(t, err, ErrShutdown)
+}

spinifex/agents/lambda/hostlink/tls.go

@@ -0,0 +1,115 @@
+package hostlink
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/pki"
+)
+
+// FwCfgDir is where the guest kernel exposes the executor's fw_cfg entries.
+const FwCfgDir = FwCfgRoot + "/by_name/" + lambdav1.FwCfgPrefix
+
+// Identity is the guest's material for one environment generation.
+type Identity struct {
+	Cert     tls.Certificate
+	HostCert *x509.Certificate
+	Scope    lambdav1.Scope
+	HostAddr string
+}
+
+// LoadIdentity reads the executor's fw_cfg entries from dir.
+func LoadIdentity(dir string) (Identity, error) {
+	read := func(name string) ([]byte, error) {
+		b, err := os.ReadFile(filepath.Join(dir, name, "raw"))
+		if err != nil {
+			return nil, fmt.Errorf("hostlink: read %s: %w", name, err)
+		}
+		return b, nil
+	}
+	var id Identity
+	keyPEM, err := read(lambdav1.EntryGuestKey)
+	if err != nil {
+		return id, err
+	}
+	certPEM, err := read(lambdav1.EntryGuestCert)
+	if err != nil {
+		return id, err
+	}
+	hostPEM, err := read(lambdav1.EntryHostCert)
+	if err != nil {
+		return id, err
+	}
+	scope, err := read(lambdav1.EntryScope)
+	if err != nil {
+		return id, err
+	}
+	addr, err := read(lambdav1.EntryHostAddr)
+	if err != nil {
+		return id, err
+	}
+
+	if id.Cert, err = tls.X509KeyPair(certPEM, keyPEM); err != nil {
+		return Identity{}, fmt.Errorf("hostlink: guest key pair: %w", err)
+	}
+	blk, rest := pem.Decode(hostPEM)
+	if blk == nil || blk.Type != "CERTIFICATE" || len(strings.TrimSpace(string(rest))) != 0 {
+		return Identity{}, errors.New("hostlink: host certificate: want exactly one PEM certificate")
+	}
+	if id.HostCert, err = x509.ParseCertificate(blk.Bytes); err != nil {
+		return Identity{}, fmt.Errorf("hostlink: host certificate: %w", err)
+	}
+	if id.Scope, err = lambdav1.ParseScope(strings.TrimSpace(string(scope))); err != nil {
+		return Identity{}, err
+	}
+	id.HostAddr = strings.TrimSpace(string(addr))
+	if _, _, err := net.SplitHostPort(id.HostAddr); err != nil {
+		return Identity{}, fmt.Errorf("hostlink: host address: %w", err)
+	}
+	return id, nil
+}
+
+// ClientConfig is the guest's TLS 1.3 configuration. The host has no DNS name,
+// so VerifyConnection checks the pinned host certificate and its URI SAN.
+func (id Identity) ClientConfig() *tls.Config {
+	return &tls.Config{
+		MinVersion:         tls.VersionTLS13,
+		MaxVersion:         tls.VersionTLS13,
+		Certificates:       []tls.Certificate{id.Cert},
+		NextProtos:         []string{lambdav1.ALPNv1},
+		InsecureSkipVerify: true, //nolint:gosec // VerifyConnection performs full validation against the pinned host certificate.
+		VerifyConnection: func(cs tls.ConnectionState) error {
+			if cs.NegotiatedProtocol != lambdav1.ALPNv1 {
+				return fmt.Errorf("hostlink: application protocol %q, want %q", cs.NegotiatedProtocol, lambdav1.ALPNv1)
+			}
+			return pki.VerifyPinned(cs.PeerCertificates, id.HostCert, x509.ExtKeyUsageServerAuth, id.Scope.URI(lambdav1.RoleHost))
+		},
+	}
+}
+
+// Dial opens the authenticated control connection to the host. Under TLS 1.3
+// the host may still refuse the guest's certificate after Dial returns; that
+// surfaces on the first read, so the caller must not treat Dial as acceptance.
+func Dial(ctx context.Context, id Identity) (*tls.Conn, error) {
+	d := tls.Dialer{NetDialer: &net.Dialer{Timeout: 10 * time.Second}, Config: id.ClientConfig()}
+	c, err := d.DialContext(ctx, "tcp", id.HostAddr)
+	if err != nil {
+		return nil, fmt.Errorf("hostlink: dial host %s: %w", id.HostAddr, err)
+	}
+	tc, ok := c.(*tls.Conn)
+	if !ok {
+		_ = c.Close()
+		return nil, fmt.Errorf("hostlink: dial host %s: unexpected connection type %T", id.HostAddr, c)
+	}
+	return tc, nil
+}

spinifex/agents/lambda/hostlink/tls_test.go

@@ -0,0 +1,291 @@
+package hostlink
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"net/url"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/pki"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	gen1 lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-1"
+	gen2 lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-2"
+)
+
+type testIdent struct {
+	cert    tls.Certificate
+	leaf    *x509.Certificate
+	certPEM []byte
+	keyPEM  []byte
+}
+
+func newTestIdent(t *testing.T, scope lambdav1.Scope, role string) testIdent {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	u, err := url.Parse(scope.URI(role))
+	require.NoError(t, err)
+	usage := x509.ExtKeyUsageServerAuth
+	if role == lambdav1.RoleGuest {
+		usage = x509.ExtKeyUsageClientAuth
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: role},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{usage},
+		BasicConstraintsValid: true, URIs: []*url.URL{u},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	require.NoError(t, err)
+	kder, err := x509.MarshalECPrivateKey(key)
+	require.NoError(t, err)
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: kder})
+	c, err := tls.X509KeyPair(certPEM, keyPEM)
+	require.NoError(t, err)
+	leaf, err := x509.ParseCertificate(der)
+	require.NoError(t, err)
+	return testIdent{cert: c, leaf: leaf, certPEM: certPEM, keyPEM: keyPEM}
+}
+
+// hostConfig is the host end as the executor will run it: it pins one guest
+// certificate and checks its URI SAN with the shared pki.VerifyPinned.
+func hostConfig(host testIdent, pinGuest *x509.Certificate, scope lambdav1.Scope) *tls.Config {
+	return &tls.Config{
+		MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{host.cert},
+		ClientAuth: tls.RequireAnyClientCert, NextProtos: []string{lambdav1.ALPNv1},
+		VerifyConnection: func(cs tls.ConnectionState) error {
+			return pki.VerifyPinned(cs.PeerCertificates, pinGuest, x509.ExtKeyUsageClientAuth, scope.URI(lambdav1.RoleGuest))
+		},
+	}
+}
+
+// serve accepts one connection with cfg, writes "ok" if the handshake
+// succeeds, and returns the address and the host's handshake result.
+func serve(t *testing.T, cfg *tls.Config) (string, <-chan error) {
+	t.Helper()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	t.Cleanup(func() { ln.Close() })
+	done := make(chan error, 1)
+	go func() {
+		raw, err := ln.Accept()
+		if err != nil {
+			done <- err
+			return
+		}
+		defer raw.Close()
+		tc := tls.Server(raw, cfg)
+		_ = tc.SetDeadline(time.Now().Add(5 * time.Second))
+		err = tc.Handshake()
+		if err == nil {
+			_, err = tc.Write([]byte("ok"))
+		}
+		done <- err
+	}()
+	return ln.Addr().String(), done
+}
+
+func guestIdentity(guest, pinHost testIdent, scope lambdav1.Scope, addr string) Identity {
+	return Identity{Cert: guest.cert, HostCert: pinHost.leaf, Scope: scope, HostAddr: addr}
+}
+
+func dialAndRead(t *testing.T, id Identity) (string, error) {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	c, err := Dial(ctx, id)
+	if err != nil {
+		return "", err
+	}
+	defer c.Close()
+	_ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
+	buf := make([]byte, 2)
+	n, err := c.Read(buf)
+	return string(buf[:n]), err
+}
+
+func TestDialMutualAuthentication(t *testing.T) {
+	host, guest := newTestIdent(t, gen1, lambdav1.RoleHost), newTestIdent(t, gen1, lambdav1.RoleGuest)
+	addr, hostErr := serve(t, hostConfig(host, guest.leaf, gen1))
+
+	got, err := dialAndRead(t, guestIdentity(guest, host, gen1, addr))
+	require.NoError(t, err)
+	assert.Equal(t, "ok", got)
+	require.NoError(t, <-hostErr)
+}
+
+func TestDialRefusesHost(t *testing.T) {
+	guest := newTestIdent(t, gen1, lambdav1.RoleGuest)
+	pinned := newTestIdent(t, gen1, lambdav1.RoleHost)
+	cases := []struct {
+		name    string
+		host    testIdent
+		pin     testIdent
+		mutate  func(*tls.Config)
+		wantErr string
+	}{
+		{"unpinned host certificate", newTestIdent(t, gen1, lambdav1.RoleHost), pinned, nil, "peer certificate"},
+		{"pinned certificate of a later generation", newTestIdent(t, gen2, lambdav1.RoleHost), testIdent{}, nil, "peer identity mismatch"},
+		{"guest role presented as host", newTestIdent(t, gen1, lambdav1.RoleGuest), testIdent{}, nil, "peer certificate"},
+		{"no application protocol", pinned, pinned, func(c *tls.Config) { c.NextProtos = nil }, "application protocol"},
+		{"TLS 1.2 only", pinned, pinned, func(c *tls.Config) { c.MinVersion, c.MaxVersion = tls.VersionTLS12, tls.VersionTLS12 }, "protocol version"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			pin := tc.pin
+			if pin.leaf == nil {
+				pin = tc.host // the guest pins exactly what is presented
+			}
+			cfg := hostConfig(tc.host, guest.leaf, gen1)
+			if tc.mutate != nil {
+				tc.mutate(cfg)
+			}
+			addr, _ := serve(t, cfg)
+			_, err := dialAndRead(t, guestIdentity(guest, pin, gen1, addr))
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.wantErr)
+		})
+	}
+}
+
+// newCASignedHost returns a CA and a host leaf it signed carrying the same URI
+// SAN as a legitimate host certificate.
+func newCASignedHost(t *testing.T, scope lambdav1.Scope) (ca testIdent, leaf tls.Certificate) {
+	t.Helper()
+	caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	u, err := url.Parse(scope.URI(lambdav1.RoleHost))
+	require.NoError(t, err)
+	caTmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "ca"},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true, IsCA: true, URIs: []*url.URL{u},
+	}
+	caDER, err := x509.CreateCertificate(rand.Reader, caTmpl, caTmpl, &caKey.PublicKey, caKey)
+	require.NoError(t, err)
+	caCert, err := x509.ParseCertificate(caDER)
+	require.NoError(t, err)
+
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "host"},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		URIs: []*url.URL{u},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, &key.PublicKey, caKey)
+	require.NoError(t, err)
+	return testIdent{leaf: caCert}, tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
+}
+
+// Trusting the pin as a root is not enough: a different leaf it signed, with
+// the right URI SAN, must still be refused.
+func TestDialRefusesLeafSignedByPin(t *testing.T) {
+	guest := newTestIdent(t, gen1, lambdav1.RoleGuest)
+	ca, leaf := newCASignedHost(t, gen1)
+	for name, chain := range map[string][][]byte{
+		"leaf only":    leaf.Certificate,
+		"leaf with CA": {leaf.Certificate[0], ca.leaf.Raw},
+	} {
+		t.Run(name, func(t *testing.T) {
+			cfg := hostConfig(testIdent{}, guest.leaf, gen1)
+			cfg.Certificates = []tls.Certificate{{Certificate: chain, PrivateKey: leaf.PrivateKey}}
+			addr, _ := serve(t, cfg)
+			_, err := dialAndRead(t, guestIdentity(guest, ca, gen1, addr))
+			require.Error(t, err)
+			assert.Regexp(t, "not the pinned certificate|want exactly the pinned one", err.Error())
+		})
+	}
+}
+
+func TestHostRefusalSurfacesOnFirstRead(t *testing.T) {
+	host, guest := newTestIdent(t, gen1, lambdav1.RoleHost), newTestIdent(t, gen1, lambdav1.RoleGuest)
+	other := newTestIdent(t, gen1, lambdav1.RoleGuest)
+	addr, hostErr := serve(t, hostConfig(host, other.leaf, gen1))
+
+	_, err := dialAndRead(t, guestIdentity(guest, host, gen1, addr))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "bad certificate")
+	assert.ErrorContains(t, <-hostErr, "peer certificate")
+}
+
+func TestHostRefusesGuestOfAnotherGeneration(t *testing.T) {
+	host, guest := newTestIdent(t, gen1, lambdav1.RoleHost), newTestIdent(t, gen2, lambdav1.RoleGuest)
+	addr, hostErr := serve(t, hostConfig(host, guest.leaf, gen1))
+
+	_, err := dialAndRead(t, guestIdentity(guest, host, gen1, addr))
+	require.Error(t, err)
+	assert.ErrorIs(t, <-hostErr, pki.ErrPeerIdentity)
+}
+
+func writeEntries(t *testing.T, entries map[string][]byte) string {
+	t.Helper()
+	dir := t.TempDir()
+	for name, b := range entries {
+		require.NoError(t, os.MkdirAll(filepath.Join(dir, name), 0o700))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, name, "raw"), b, 0o600))
+	}
+	return dir
+}
+
+func validEntries(t *testing.T) map[string][]byte {
+	host, guest := newTestIdent(t, gen1, lambdav1.RoleHost), newTestIdent(t, gen1, lambdav1.RoleGuest)
+	return map[string][]byte{
+		lambdav1.EntryGuestKey: guest.keyPEM, lambdav1.EntryGuestCert: guest.certPEM, lambdav1.EntryHostCert: host.certPEM,
+		lambdav1.EntryScope: []byte(string(gen1) + "\n"), lambdav1.EntryHostAddr: []byte("10.15.8.1:18443\n"),
+	}
+}
+
+func TestLoadIdentity(t *testing.T) {
+	id, err := LoadIdentity(writeEntries(t, validEntries(t)))
+	require.NoError(t, err)
+	assert.Equal(t, gen1, id.Scope)
+	assert.Equal(t, "10.15.8.1:18443", id.HostAddr)
+	require.Len(t, id.HostCert.URIs, 1)
+	assert.Equal(t, gen1.URI(lambdav1.RoleHost), id.HostCert.URIs[0].String())
+}
+
+func TestLoadIdentityRejects(t *testing.T) {
+	other := newTestIdent(t, gen1, lambdav1.RoleGuest)
+	cases := []struct {
+		name    string
+		edit    func(map[string][]byte)
+		wantErr string
+	}{
+		{"missing entry", func(e map[string][]byte) { delete(e, lambdav1.EntryHostCert) }, "read host-cert"},
+		{"key does not match certificate", func(e map[string][]byte) { e[lambdav1.EntryGuestKey] = other.keyPEM }, "guest key pair"},
+		{"two host certificates", func(e map[string][]byte) {
+			e[lambdav1.EntryHostCert] = append(e[lambdav1.EntryHostCert], e[lambdav1.EntryHostCert]...)
+		}, "exactly one PEM"},
+		{"host certificate not PEM", func(e map[string][]byte) { e[lambdav1.EntryHostCert] = []byte("nope") }, "exactly one PEM"},
+		{"short scope", func(e map[string][]byte) { e[lambdav1.EntryScope] = []byte("ap-southeast-2/000000000001/fn-1") }, "want 5 segments"},
+		{"address without port", func(e map[string][]byte) { e[lambdav1.EntryHostAddr] = []byte("10.15.8.1") }, "host address"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			e := validEntries(t)
+			tc.edit(e)
+			_, err := LoadIdentity(writeEntries(t, e))
+			assert.ErrorContains(t, err, tc.wantErr)
+		})
+	}
+}

spinifex/agents/lambda/runtimeapi/server.go

@@ -0,0 +1,285 @@
+// Package runtimeapi serves the AWS Lambda Runtime API (version 2018-06-01)
+// inside the guest, handing one invocation at a time to the function runtime.
+package runtimeapi
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// MaxResponseBytes is the largest response payload a runtime may post.
+const MaxResponseBytes = 6291456
+
+const (
+	// ErrTypeResponseTooLarge is the function error for an oversize response.
+	ErrTypeResponseTooLarge = "Function.ResponseSizeTooLarge"
+	// ErrTypeUnhandled is the function error when the runtime names no type.
+	ErrTypeUnhandled = "Unhandled"
+
+	errTypeInvalidRequestID = "InvalidRequestID"
+	errTypeTooLarge         = "RequestEntityTooLarge"
+	errTypeInvalidRequest   = "InvalidRequest"
+
+	headerRequestID = "Lambda-Runtime-Aws-Request-Id"
+	headerDeadline  = "Lambda-Runtime-Deadline-Ms"
+	headerARN       = "Lambda-Runtime-Invoked-Function-Arn"
+	headerTraceID   = "Lambda-Runtime-Trace-Id"
+	headerErrorType = "Lambda-Runtime-Function-Error-Type"
+
+	maxInitErrorBytes = 1 << 20
+)
+
+var (
+	// ErrBusy is returned by Invoke while another invocation is in flight.
+	ErrBusy = errors.New("runtimeapi: an invocation is already in flight")
+	// ErrDeadlineExceeded is returned by Invoke when the deadline passes.
+	ErrDeadlineExceeded = errors.New("runtimeapi: invocation deadline exceeded")
+	// ErrInvalidInvocation is returned by Invoke for an invocation with no request ID.
+	ErrInvalidInvocation = errors.New("runtimeapi: invocation has no request id")
+)
+
+// Invocation is one event handed to the runtime.
+type Invocation struct {
+	RequestID   string
+	FunctionARN string
+	Deadline    time.Time
+	TraceID     string
+	Payload     []byte
+}
+
+// Result is what the runtime answered. FunctionError is empty on success and
+// otherwise the error type.
+type Result struct {
+	Payload       []byte
+	FunctionError string
+}
+
+// InitError is an initialisation failure reported by the runtime.
+type InitError struct {
+	Type string
+	Body []byte
+}
+
+type flight struct {
+	inv   Invocation
+	taken bool
+	done  chan Result
+}
+
+// Server implements the Runtime API for a single in-flight invocation.
+type Server struct {
+	mu   sync.Mutex
+	cur  *flight
+	wake chan struct{}
+
+	initOnce sync.Once
+	initDone chan struct{}
+	initErr  InitError
+}
+
+// New returns a Server with no invocation in flight.
+func New() *Server {
+	return &Server{wake: make(chan struct{}, 1), initDone: make(chan struct{})}
+}
+
+// InitFailed is closed once the runtime reports an init error.
+func (s *Server) InitFailed() <-chan struct{} { return s.initDone }
+
+// InitError returns the reported init error, if any.
+func (s *Server) InitError() (InitError, bool) {
+	select {
+	case <-s.initDone:
+		return s.initErr, true
+	default:
+		return InitError{}, false
+	}
+}
+
+// Invoke hands inv to the runtime and blocks until the runtime posts a
+// response or error, the deadline passes, or ctx ends.
+func (s *Server) Invoke(ctx context.Context, inv Invocation) (Result, error) {
+	if inv.RequestID == "" {
+		return Result{}, ErrInvalidInvocation
+	}
+	f := &flight{inv: inv, done: make(chan Result, 1)}
+	s.mu.Lock()
+	if s.cur != nil {
+		s.mu.Unlock()
+		return Result{}, ErrBusy
+	}
+	s.cur = f
+	s.mu.Unlock()
+	select {
+	case s.wake <- struct{}{}:
+	default:
+	}
+
+	timer := time.NewTimer(time.Until(inv.Deadline))
+	defer timer.Stop()
+	var cause error
+	select {
+	case res := <-f.done:
+		return res, nil
+	case <-timer.C:
+		cause = ErrDeadlineExceeded
+	case <-ctx.Done():
+		cause = ctx.Err()
+	}
+
+	s.mu.Lock()
+	if s.cur == f {
+		s.cur = nil
+		s.mu.Unlock()
+		return Result{}, cause
+	}
+	s.mu.Unlock()
+	// Completed while we were giving up; the result is already queued.
+	return <-f.done, nil
+}
+
+// Handler returns the Runtime API routes.
+func (s *Server) Handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /2018-06-01/runtime/invocation/next", s.next)
+	mux.HandleFunc("POST /2018-06-01/runtime/invocation/{id}/response", s.response)
+	mux.HandleFunc("POST /2018-06-01/runtime/invocation/{id}/error", s.invocationError)
+	mux.HandleFunc("POST /2018-06-01/runtime/init/error", s.initError)
+	return mux
+}
+
+func (s *Server) next(w http.ResponseWriter, r *http.Request) {
+	for {
+		s.mu.Lock()
+		f := s.cur
+		if f != nil && !f.taken {
+			f.taken = true
+			s.mu.Unlock()
+			h := w.Header()
+			h.Set(headerRequestID, f.inv.RequestID)
+			h.Set(headerDeadline, strconv.FormatInt(f.inv.Deadline.UnixMilli(), 10))
+			h.Set(headerARN, f.inv.FunctionARN)
+			if f.inv.TraceID != "" {
+				h.Set(headerTraceID, f.inv.TraceID)
+			}
+			_, _ = w.Write(f.inv.Payload)
+			return
+		}
+		s.mu.Unlock()
+		select {
+		case <-s.wake:
+		case <-r.Context().Done():
+			return
+		}
+	}
+}
+
+// claimable reports whether id names the delivered, uncompleted invocation.
+func (s *Server) claimable(id string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.cur != nil && s.cur.taken && s.cur.inv.RequestID == id
+}
+
+// complete finishes the invocation named id, reporting whether it was live.
+func (s *Server) complete(id string, res Result) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	f := s.cur
+	if f == nil || !f.taken || f.inv.RequestID != id {
+		return false
+	}
+	s.cur = nil
+	f.done <- res
+	return true
+}
+
+func (s *Server) response(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	if !s.claimable(id) {
+		invalidID(w, id)
+		return
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, MaxResponseBytes+1))
+	if err != nil {
+		writeError(w, http.StatusBadRequest, errTypeInvalidRequest, "read body: "+err.Error())
+		return
+	}
+	if len(body) > MaxResponseBytes {
+		msg := "Response payload size exceeded maximum allowed payload size (" + strconv.Itoa(MaxResponseBytes) + " bytes)."
+		s.complete(id, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeResponseTooLarge})
+		writeError(w, http.StatusRequestEntityTooLarge, errTypeTooLarge, msg)
+		return
+	}
+	if !s.complete(id, Result{Payload: body}) {
+		invalidID(w, id)
+		return
+	}
+	writeStatus(w)
+}
+
+func (s *Server) invocationError(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	if !s.claimable(id) {
+		invalidID(w, id)
+		return
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, MaxResponseBytes))
+	if err != nil {
+		writeError(w, http.StatusBadRequest, errTypeInvalidRequest, "read body: "+err.Error())
+		return
+	}
+	errType := r.Header.Get(headerErrorType)
+	if errType == "" {
+		errType = ErrTypeUnhandled
+	}
+	if !s.complete(id, Result{Payload: body, FunctionError: errType}) {
+		invalidID(w, id)
+		return
+	}
+	writeStatus(w)
+}
+
+func (s *Server) initError(w http.ResponseWriter, r *http.Request) {
+	body, _ := io.ReadAll(io.LimitReader(r.Body, maxInitErrorBytes))
+	s.initOnce.Do(func() {
+		s.initErr = InitError{Type: r.Header.Get(headerErrorType), Body: body}
+		close(s.initDone)
+	})
+	writeStatus(w)
+}
+
+func invalidID(w http.ResponseWriter, id string) {
+	writeError(w, http.StatusBadRequest, errTypeInvalidRequestID, "Invalid request ID: "+id)
+}
+
+func writeStatus(w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusAccepted)
+	_, _ = w.Write([]byte(`{"status":"OK"}`))
+}
+
+func writeError(w http.ResponseWriter, code int, errType, msg string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(code)
+	_, _ = w.Write(ErrorBody(msg, errType))
+}
+
+type errorPayload struct {
+	ErrorMessage string `json:"errorMessage"`
+	ErrorType    string `json:"errorType"`
+}
+
+// ErrorBody renders the Lambda error document for msg and errType.
+func ErrorBody(msg, errType string) []byte {
+	b, err := json.Marshal(errorPayload{ErrorMessage: msg, ErrorType: errType})
+	if err != nil {
+		return []byte(`{"errorType":"Runtime.Unknown"}`)
+	}
+	return b
+}

spinifex/agents/lambda/runtimeapi/server_test.go

@@ -0,0 +1,292 @@
+package runtimeapi
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const base = "/2018-06-01/runtime/"
+
+type invokeOut struct {
+	res Result
+	err error
+}
+
+func start(t *testing.T) (*Server, *httptest.Server) {
+	t.Helper()
+	s := New()
+	ts := httptest.NewServer(s.Handler())
+	t.Cleanup(ts.Close)
+	return s, ts
+}
+
+func invokeAsync(s *Server, inv Invocation) <-chan invokeOut {
+	ch := make(chan invokeOut, 1)
+	go func() {
+		res, err := s.Invoke(context.Background(), inv)
+		ch <- invokeOut{res, err}
+	}()
+	return ch
+}
+
+func post(t *testing.T, url string, body []byte, hdr map[string]string) (*http.Response, []byte) {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
+	require.NoError(t, err)
+	for k, v := range hdr {
+		req.Header.Set(k, v)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(resp.Body)
+	return resp, b
+}
+
+func get(t *testing.T, url string) (*http.Response, []byte) {
+	t.Helper()
+	resp, err := http.Get(url)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(resp.Body)
+	return resp, b
+}
+
+func inv(id string, d time.Duration) Invocation {
+	return Invocation{
+		RequestID:   id,
+		FunctionARN: "arn:aws:lambda:ap-southeast-2:000000000001:function:f",
+		Deadline:    time.Now().Add(d),
+		Payload:     []byte(`{"a":1}`),
+	}
+}
+
+func errType(t *testing.T, body []byte) string {
+	t.Helper()
+	var e map[string]string
+	require.NoError(t, json.Unmarshal(body, &e))
+	assert.NotEmpty(t, e["errorMessage"])
+	return e["errorType"]
+}
+
+func TestNextResponseRoundTrip(t *testing.T) {
+	s, ts := start(t)
+	in := inv("req-1", 2*time.Second)
+	in.TraceID = "Root=1-abc"
+	out := invokeAsync(s, in)
+
+	resp, body := get(t, ts.URL+base+"invocation/next")
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, `{"a":1}`, string(body))
+	assert.Equal(t, "req-1", resp.Header.Get("Lambda-Runtime-Aws-Request-Id"))
+	assert.Equal(t, strconv.FormatInt(in.Deadline.UnixMilli(), 10), resp.Header.Get("Lambda-Runtime-Deadline-Ms"))
+	assert.Equal(t, in.FunctionARN, resp.Header.Get("Lambda-Runtime-Invoked-Function-Arn"))
+	assert.Equal(t, "Root=1-abc", resp.Header.Get("Lambda-Runtime-Trace-Id"))
+
+	r, _ := post(t, ts.URL+base+"invocation/req-1/response", []byte(`{"sum":3}`), nil)
+	assert.Equal(t, http.StatusAccepted, r.StatusCode)
+
+	got := <-out
+	require.NoError(t, got.err)
+	assert.Equal(t, `{"sum":3}`, string(got.res.Payload))
+	assert.Empty(t, got.res.FunctionError)
+}
+
+func TestTraceHeaderOmittedWhenUnset(t *testing.T) {
+	s, ts := start(t)
+	invokeAsync(s, inv("req-1", time.Second))
+	resp, _ := get(t, ts.URL+base+"invocation/next")
+	_, present := resp.Header["Lambda-Runtime-Trace-Id"]
+	assert.False(t, present)
+}
+
+func TestErrorPostKeepsBodyAndType(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+
+	body := []byte(`{"errorMessage":"boom","errorType":"Custom"}`)
+	r, _ := post(t, ts.URL+base+"invocation/req-1/error", body, map[string]string{"Lambda-Runtime-Function-Error-Type": "Custom"})
+	assert.Equal(t, http.StatusAccepted, r.StatusCode)
+
+	got := <-out
+	require.NoError(t, got.err)
+	assert.Equal(t, "Custom", got.res.FunctionError)
+	assert.JSONEq(t, string(body), string(got.res.Payload))
+}
+
+func TestErrorPostWithoutTypeIsUnhandled(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+	post(t, ts.URL+base+"invocation/req-1/error", []byte(`{}`), nil)
+	assert.Equal(t, "Unhandled", (<-out).res.FunctionError)
+}
+
+func TestUnknownAndRepeatedIDRefused(t *testing.T) {
+	s, ts := start(t)
+
+	r, b := post(t, ts.URL+base+"invocation/nope/response", []byte("x"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+	assert.Equal(t, "InvalidRequestID", errType(t, b))
+
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+	post(t, ts.URL+base+"invocation/req-1/response", []byte("ok"), nil)
+	<-out
+
+	r, b = post(t, ts.URL+base+"invocation/req-1/response", []byte("again"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+	assert.Equal(t, "InvalidRequestID", errType(t, b))
+	r, _ = post(t, ts.URL+base+"invocation/req-1/error", []byte("{}"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+}
+
+func TestResponseNotAcceptedBeforeDelivery(t *testing.T) {
+	s, ts := start(t)
+	invokeAsync(s, inv("req-1", time.Second))
+	r, _ := post(t, ts.URL+base+"invocation/req-1/response", []byte("early"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+}
+
+func TestOversizeResponse(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 5*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+
+	r, b := post(t, ts.URL+base+"invocation/req-1/response", make([]byte, MaxResponseBytes+1), nil)
+	assert.Equal(t, http.StatusRequestEntityTooLarge, r.StatusCode)
+	assert.Equal(t, "RequestEntityTooLarge", errType(t, b))
+
+	got := <-out
+	require.NoError(t, got.err)
+	assert.Equal(t, "Function.ResponseSizeTooLarge", got.res.FunctionError)
+	assert.Equal(t, "Function.ResponseSizeTooLarge", errType(t, got.res.Payload))
+}
+
+func TestResponseAtLimitAccepted(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 5*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+	r, _ := post(t, ts.URL+base+"invocation/req-1/response", make([]byte, MaxResponseBytes), nil)
+	assert.Equal(t, http.StatusAccepted, r.StatusCode)
+	got := <-out
+	require.NoError(t, got.err)
+	assert.Len(t, got.res.Payload, MaxResponseBytes)
+}
+
+func TestDeadlineExpiryRefusesLateResponse(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 40*time.Millisecond))
+	get(t, ts.URL+base+"invocation/next")
+
+	got := <-out
+	require.ErrorIs(t, got.err, ErrDeadlineExceeded)
+
+	r, b := post(t, ts.URL+base+"invocation/req-1/response", []byte("late"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+	assert.Equal(t, "InvalidRequestID", errType(t, b))
+
+	// The slot is free again for the next invocation.
+	out = invokeAsync(s, inv("req-2", 2*time.Second))
+	resp, _ := get(t, ts.URL+base+"invocation/next")
+	assert.Equal(t, "req-2", resp.Header.Get("Lambda-Runtime-Aws-Request-Id"))
+	post(t, ts.URL+base+"invocation/req-2/response", []byte("ok"), nil)
+	require.NoError(t, (<-out).err)
+}
+
+func TestConcurrentInvokeRefused(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	require.Eventually(t, func() bool {
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		return s.cur != nil
+	}, time.Second, time.Millisecond)
+
+	_, err := s.Invoke(context.Background(), inv("req-2", time.Second))
+	require.ErrorIs(t, err, ErrBusy)
+
+	get(t, ts.URL+base+"invocation/next")
+	post(t, ts.URL+base+"invocation/req-1/response", []byte("ok"), nil)
+	require.NoError(t, (<-out).err)
+}
+
+func TestInvokeContextCancel(t *testing.T) {
+	s, _ := start(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() {
+		_, err := s.Invoke(ctx, inv("req-1", time.Minute))
+		done <- err
+	}()
+	cancel()
+	require.ErrorIs(t, <-done, context.Canceled)
+}
+
+func TestNextBlocksUntilInvoke(t *testing.T) {
+	s, ts := start(t)
+	type got struct {
+		resp *http.Response
+		body []byte
+	}
+	ch := make(chan got, 1)
+	go func() {
+		resp, err := http.Get(ts.URL + base + "invocation/next") //nolint:noctx // test client
+		if err != nil {
+			ch <- got{}
+			return
+		}
+		defer resp.Body.Close()
+		body, _ := io.ReadAll(resp.Body)
+		ch <- got{resp, body}
+	}()
+
+	select {
+	case <-ch:
+		t.Fatal("next returned before any invocation")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	g := <-ch
+	require.NotNil(t, g.resp, "next request failed")
+	assert.Equal(t, "req-1", g.resp.Header.Get("Lambda-Runtime-Aws-Request-Id"))
+	post(t, ts.URL+base+"invocation/req-1/response", []byte("ok"), nil)
+	require.NoError(t, (<-out).err)
+}
+
+func TestInitErrorRecorded(t *testing.T) {
+	s, ts := start(t)
+	_, ok := s.InitError()
+	require.False(t, ok)
+
+	r, _ := post(t, ts.URL+base+"init/error", []byte(`{"errorMessage":"bad"}`), map[string]string{"Lambda-Runtime-Function-Error-Type": "Runtime.InitError"})
+	assert.Equal(t, http.StatusAccepted, r.StatusCode)
+
+	select {
+	case <-s.InitFailed():
+	default:
+		t.Fatal("InitFailed not closed")
+	}
+	ie, ok := s.InitError()
+	require.True(t, ok)
+	assert.Equal(t, "Runtime.InitError", ie.Type)
+	assert.JSONEq(t, `{"errorMessage":"bad"}`, string(ie.Body))
+}
+
+func TestInvokeRequiresRequestID(t *testing.T) {
+	s := New()
+	_, err := s.Invoke(context.Background(), Invocation{Deadline: time.Now().Add(time.Second)})
+	require.ErrorIs(t, err, ErrInvalidInvocation)
+}

spinifex/domains/acm/keyalgorithm_test.go

@@ -152,3 +152,30 @@ func TestImportCertificate_SerialZeroRejected(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorValidationException, code)
 	assert.Equal(t, "The serial number in the certificate is not supported by ACM.", msg)
 }
+
+func TestImportCertificate_ExpiredRejected(t *testing.T) {
+	t.Parallel()
+	svc := setupACMService(t)
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	require.NoError(t, err)
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(time.Now().UnixNano()),
+		Subject:      pkix.Name{CommonName: "expired.example.com"},
+		NotBefore:    time.Now().Add(-48 * time.Hour),
+		NotAfter:     time.Now().Add(-24 * time.Hour),
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	require.NoError(t, err)
+	_, err = svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
+		Certificate: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}),
+		PrivateKey:  pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}),
+	}, testAccountID)
+	code, msg, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorValidationException, code)
+	assert.Equal(t, "The certificate is expired. You can Import a certificate only during its validity period.", msg)
+
+	list, err := svc.ListCertificates(context.Background(), &acm.ListCertificatesInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, list.CertificateSummaryList)
+}

spinifex/domains/acm/service_impl.go

@@ -223,10 +223,13 @@ func (s *ACMServiceImpl) ImportCertificate(ctx context.Context, input *acm.Impor
 	if leaf.SerialNumber.Sign() == 0 {
 		return nil, awserrors.Errorf(awserrors.ErrorValidationException, "The serial number in the certificate is not supported by ACM.")
 	}
-
 	certArn := aws.StringValue(input.CertificateArn)
-	var inUseBy []string
 	now := time.Now().UTC()
+	if certArn == "" && now.After(leaf.NotAfter) {
+		return nil, awserrors.Errorf(awserrors.ErrorValidationException, "The certificate is expired. You can Import a certificate only during its validity period.")
+	}
+
+	var inUseBy []string
 	createdAt := now
 	tags := tagsToMap(input.Tags)
 	reimport := certArn != ""

spinifex/domains/ec2/awsapi/actions.go

@@ -134,6 +134,9 @@ func marshalEC2Response(action string, output any) ([]byte, error) {
 
 var ec2Actions = map[string]ec2Action{
 	"DescribeInstances": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstancesInput, c *call, accountID string) (any, error) {
+		if err := ec2instanceapi.ValidateDescribeInstancesInput(input); err != nil {
+			return nil, err
+		}
 		out, err := ec2instanceapi.DescribeInstancesChecked(ctx, input, c.NATS, 0, c.NodeIDs, accountID)
 		if err != nil {
 			return out, err
@@ -257,6 +260,9 @@ var ec2Actions = map[string]ec2Action{
 		}
 	}),
 	"DescribeImages": ec2Handler(func(ctx context.Context, input *ec2.DescribeImagesInput, c *call, accountID string) (any, error) {
+		if err := ec2imageapi.ValidateDescribeImagesInput(input); err != nil {
+			return nil, err
+		}
 		return ec2imageapi.DescribeImages(ctx, input, c.NATS, accountID)
 	}),
 	"CreateImage": ec2Handler(func(ctx context.Context, input *ec2.CreateImageInput, c *call, accountID string) (any, error) {
@@ -287,6 +293,9 @@ var ec2Actions = map[string]ec2Action{
 		return ec2zoneapi.DescribeAvailabilityZones(input, c.Region, c.AZ)
 	}),
 	"DescribeVolumes": ec2Handler(func(ctx context.Context, input *ec2.DescribeVolumesInput, c *call, accountID string) (any, error) {
+		if err := ec2volumeapi.ValidateDescribeVolumesInput(input); err != nil {
+			return nil, err
+		}
 		return ec2volumeapi.DescribeVolumes(ctx, input, c.NATS, accountID)
 	}),
 	"ModifyVolume": ec2Handler(func(ctx context.Context, input *ec2.ModifyVolumeInput, c *call, accountID string) (any, error) {

spinifex/domains/ec2/awsapi/actions_test.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -30,3 +31,27 @@ func TestImportKeyPair_Base64PaddingWorkaround(t *testing.T) {
 	assert.NotContains(t, q["PublicKeyMaterial"], "%3D",
 		"Expected no URL-encoded padding remaining")
 }
+
+// A well-prefixed ID of the wrong shape is refused before any service is
+// asked, so it can never come back as NotFound.
+func TestDescribe_MalformedIDRejectedBeforeLookup(t *testing.T) {
+	tests := []struct {
+		action, param, id, code, msg string
+	}{
+		{"DescribeInstances", "InstanceId.1", "i-zzz", awserrors.ErrorInvalidInstanceIDMalformed, `Invalid id: "i-zzz"`},
+		{"DescribeImages", "ImageId.1", "ami-zzz", awserrors.ErrorInvalidAMIIDMalformed, `Invalid id: "ami-zzz"`},
+		{"DescribeVolumes", "VolumeId.1", "vol-zzz", awserrors.ErrorInvalidParameterValue, "Value (vol-zzz) for parameter volumes is invalid. Expected: 'vol-...'."},
+	}
+	for _, tt := range tests {
+		t.Run(tt.action, func(t *testing.T) {
+			h := ec2Actions[tt.action]
+			input, err := h.parse(map[string]string{tt.param: tt.id})
+			require.NoError(t, err)
+			_, err = h.dispatch(t.Context(), tt.action, input, &call{Handler: NewHandler(Deps{})}, "123456789012")
+			code, msg, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok)
+			assert.Equal(t, tt.code, code)
+			assert.Equal(t, tt.msg, msg)
+		})
+	}
+}

spinifex/domains/ec2/awsapi/image/DescribeImages.go

@@ -5,6 +5,7 @@ package image
 import (
 	"context"
 	"errors"
+	"regexp"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -13,8 +14,12 @@ import (
 	"github.com/nats-io/nats.go"
 )
 
-// ValidateDescribeImagesInput accepts a nil input and rejects any ImageIds entry without the ami-
-// prefix with InvalidAMIID.Malformed.
+// imageIDSuffixPattern is the AWS AMI ID shape after ami-: 8 (legacy) or 17
+// lowercase hex characters.
+var imageIDSuffixPattern = regexp.MustCompile(`^([0-9a-f]{8}|[0-9a-f]{17})$`)
+
+// ValidateDescribeImagesInput accepts a nil input and rejects any ImageIds entry that is not
+// ami- followed by 8 or 17 lowercase hex characters with InvalidAMIID.Malformed.
 func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
 	if input == nil {
 		return nil
@@ -22,9 +27,16 @@ func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
 
 	if input.ImageIds != nil {
 		for _, imageId := range input.ImageIds {
-			if imageId != nil && !strings.HasPrefix(*imageId, "ami-") {
+			if imageId == nil {
+				continue
+			}
+			suffix, ok := strings.CutPrefix(*imageId, "ami-")
+			if !ok {
 				return errors.New(awserrors.ErrorInvalidAMIIDMalformed)
 			}
+			if !imageIDSuffixPattern.MatchString(suffix) {
+				return awserrors.Errorf(awserrors.ErrorInvalidAMIIDMalformed, "Invalid id: %q", *imageId)
+			}
 		}
 	}
 
@@ -34,11 +46,6 @@ func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
 // DescribeImages implements the EC2 DescribeImages action, returning the AMIs visible to
 // accountID from the NATS image service.
 func DescribeImages(ctx context.Context, input *ec2.DescribeImagesInput, natsConn *nats.Conn, accountID string) (output ec2.DescribeImagesOutput, err error) {
-	err = ValidateDescribeImagesInput(input)
-	if err != nil {
-		return output, err
-	}
-
 	imageService := ec2image.NewNATSImageService(natsConn, 0)
 	result, err := imageService.DescribeImages(ctx, input, accountID)
 	if err != nil {

spinifex/domains/ec2/awsapi/image/DescribeImages_test.go

@@ -53,8 +53,8 @@ func TestValidateDescribeImagesInput(t *testing.T) {
 			name: "MultipleValidImageIds",
 			input: &ec2.DescribeImagesInput{
 				ImageIds: []*string{
-					aws.String("ami-111"),
-					aws.String("ami-222"),
+					aws.String("ami-1a2b3c4d"),
+					aws.String("ami-0123456789abcdef1"),
 				},
 			},
 			wantErr: false,
@@ -63,7 +63,7 @@ func TestValidateDescribeImagesInput(t *testing.T) {
 			name: "MixedValidAndInvalidImageIds",
 			input: &ec2.DescribeImagesInput{
 				ImageIds: []*string{
-					aws.String("ami-valid"),
+					aws.String("ami-0123456789abcdef0"),
 					aws.String("invalid-id"),
 				},
 			},
@@ -85,11 +85,24 @@ func TestValidateDescribeImagesInput(t *testing.T) {
 			err := ValidateDescribeImagesInput(tt.input)
 
 			if tt.wantErr {
-				assert.Error(t, err)
-				assert.Equal(t, tt.errMsg, err.Error())
+				code, ok := awserrors.ResolveErrorCode(err)
+				assert.True(t, ok)
+				assert.Equal(t, tt.errMsg, code)
 			} else {
 				assert.NoError(t, err)
 			}
 		})
 	}
 }
+
+// AWS answers a well-prefixed ID whose suffix is not 8 or 17 lowercase hex
+// characters with Malformed, not NotFound.
+func TestValidateDescribeImagesInput_MalformedSuffix(t *testing.T) {
+	for _, id := range []string{"ami-zzz", "ami-0123456789ABCDEF0", "ami-0123456789abcdef", "ami-"} {
+		err := ValidateDescribeImagesInput(&ec2.DescribeImagesInput{ImageIds: []*string{aws.String(id)}})
+		code, msg, ok := awserrors.ResolveErrorDetail(err)
+		assert.True(t, ok, id)
+		assert.Equal(t, awserrors.ErrorInvalidAMIIDMalformed, code, id)
+		assert.Equal(t, `Invalid id: "`+id+`"`, msg, id)
+	}
+}

spinifex/domains/ec2/awsapi/image/handler_test.go

@@ -39,13 +39,6 @@ func TestCreateImage_NilNATS(t *testing.T) {
 	assert.Error(t, err)
 }
 
-func TestDescribeImages_ValidationErrors(t *testing.T) {
-	_, err := DescribeImages(context.Background(), &ec2.DescribeImagesInput{
-		ImageIds: []*string{aws.String("bad-id")},
-	}, nil, "")
-	assert.EqualError(t, err, awserrors.ErrorInvalidAMIIDMalformed)
-}
-
 func TestDescribeImages_NilNATS(t *testing.T) {
 	_, err := DescribeImages(context.Background(), nil, nil, "acct-123")
 	assert.Error(t, err)

spinifex/domains/ec2/awsapi/instance/DescribeInstances.go

@@ -89,6 +89,12 @@ func DescribeInstances(ctx context.Context, input *ec2.DescribeInstancesInput, n
 	return &ec2.DescribeInstancesOutput{Reservations: reservations}, nil
 }
 
+// ValidateDescribeInstancesInput rejects any InstanceIds entry that is not a
+// well-formed instance ID with InvalidInstanceID.Malformed.
+func ValidateDescribeInstancesInput(input *ec2.DescribeInstancesInput) error {
+	return validateInstanceIDFormat(input.InstanceIds)
+}
+
 // DescribeInstancesChecked is the customer-facing variant served by the
 // gateway's DescribeInstances action. On top of DescribeInstances' aggregation,
 // it asserts InvalidInstanceID.NotFound for any explicitly named instance ID

spinifex/domains/ec2/awsapi/instance/IamInstanceProfileAssociation.go

@@ -35,7 +35,7 @@ func resolveAndAuthorizeProfile(spec *ec2.IamInstanceProfileSpecification, iamSv
 	}
 	profile, err := iamSvc.ResolveInstanceProfile(accountID, nameOrARN)
 	if err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return nil, errors.New(awserrors.ErrorInvalidIamInstanceProfileNotFound)
 		}
 		return nil, err

spinifex/domains/ec2/awsapi/instance/IamInstanceProfileAssociation_test.go

@@ -45,7 +45,7 @@ func (f *fakeIAMService) ResolveInstanceProfile(accountID, nameOrARN string) (*i
 	if f.resolveFn != nil {
 		return f.resolveFn(accountID, nameOrARN)
 	}
-	return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+	return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", nameOrARN)
 }
 
 func (f *fakeIAMService) CanonicalResourceARN(accountID string, kind arn.IAMResourceType, name string) (string, error) {

spinifex/domains/ec2/awsapi/instance/instanceid.go

@@ -25,7 +25,7 @@ var instanceIDPattern = regexp.MustCompile(`^i-([0-9a-f]{8}|[0-9a-f]{17})$`)
 func validateInstanceIDFormat(ids []*string) error {
 	for _, id := range ids {
 		if id != nil && !instanceIDPattern.MatchString(*id) {
-			return errors.New(awserrors.ErrorInvalidInstanceIDMalformed)
+			return awserrors.Errorf(awserrors.ErrorInvalidInstanceIDMalformed, "Invalid id: %q", *id)
 		}
 	}
 	return nil

spinifex/domains/ec2/awsapi/instance/instanceid_test.go

@@ -57,8 +57,10 @@ func TestValidateInstanceIDFormat(t *testing.T) {
 	}
 	for _, id := range malformed {
 		err := validateInstanceIDFormat([]*string{aws.String(id)})
-		require.Error(t, err, id)
-		assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, err.Error(), id)
+		code, msg, ok := awserrors.ResolveErrorDetail(err)
+		require.True(t, ok, id)
+		assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, code, id)
+		assert.Equal(t, `Invalid id: "`+id+`"`, msg, id)
 	}
 	err := validateInstanceIDFormat([]*string{aws.String("i-0123456789abcdef0"), aws.String("i-abc")})
 	require.Error(t, err, "one malformed id fails the whole batch")
@@ -100,8 +102,9 @@ func TestLifecycleVerbs_MalformedIDRejectedBeforeNATS(t *testing.T) {
 	}
 	for name, call := range calls {
 		err := call()
-		require.Error(t, err, name)
-		assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, err.Error(), name)
+		code, ok := awserrors.ResolveErrorCode(err)
+		require.True(t, ok, name)
+		assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, code, name)
 	}
 	require.NoError(t, nc.Flush())
 	assert.Zero(t, requests.Load(), "no NATS request may be sent for a batch holding a malformed id")

spinifex/domains/ec2/awsapi/volume/DescribeVolumes.go

@@ -5,6 +5,7 @@ package volume
 import (
 	"context"
 	"errors"
+	"regexp"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -13,8 +14,13 @@ import (
 	"github.com/nats-io/nats.go"
 )
 
-// ValidateDescribeVolumesInput accepts a nil input and rejects any VolumeIds entry without the
-// vol- prefix with InvalidVolumeID.Malformed.
+// volumeIDSuffixPattern is the AWS volume ID shape after vol-: 8 (legacy) or
+// 17 lowercase hex characters.
+var volumeIDSuffixPattern = regexp.MustCompile(`^([0-9a-f]{8}|[0-9a-f]{17})$`)
+
+// ValidateDescribeVolumesInput accepts a nil input, rejects any VolumeIds entry without the
+// vol- prefix with InvalidVolumeID.Malformed, and any with a malformed suffix with
+// InvalidParameterValue, as AWS does.
 func ValidateDescribeVolumesInput(input *ec2.DescribeVolumesInput) error {
 	if input == nil {
 		return nil
@@ -22,9 +28,17 @@ func ValidateDescribeVolumesInput(input *ec2.DescribeVolumesInput) error {
 
 	if input.VolumeIds != nil {
 		for _, volumeId := range input.VolumeIds {
-			if volumeId != nil && !strings.HasPrefix(*volumeId, "vol-") {
+			if volumeId == nil {
+				continue
+			}
+			suffix, ok := strings.CutPrefix(*volumeId, "vol-")
+			if !ok {
 				return errors.New(awserrors.ErrorInvalidVolumeIDMalformed)
 			}
+			if !volumeIDSuffixPattern.MatchString(suffix) {
+				return awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+					"Value (%s) for parameter volumes is invalid. Expected: 'vol-...'.", *volumeId)
+			}
 		}
 	}
 
@@ -35,11 +49,6 @@ func ValidateDescribeVolumesInput(input *ec2.DescribeVolumesInput) error {
 func DescribeVolumes(ctx context.Context, input *ec2.DescribeVolumesInput, natsConn *nats.Conn, accountID string) (ec2.DescribeVolumesOutput, error) {
 	var output ec2.DescribeVolumesOutput
 
-	err := ValidateDescribeVolumesInput(input)
-	if err != nil {
-		return output, err
-	}
-
 	volumeService := ec2volume.NewNATSVolumeService(natsConn)
 	result, err := volumeService.DescribeVolumes(ctx, input, accountID)
 	if err != nil {

spinifex/domains/ec2/awsapi/volume/DescribeVolumes_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -36,9 +37,9 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
 			name: "ValidMultipleVolumeIds",
 			input: &ec2.DescribeVolumesInput{
 				VolumeIds: []*string{
-					aws.String("vol-abc123"),
-					aws.String("vol-def456"),
-					aws.String("vol-ghi789"),
+					aws.String("vol-0123456789abcdef1"),
+					aws.String("vol-1a2b3c4d"),
+					aws.String("vol-0123456789abcdef2"),
 				},
 			},
 			wantErr: false,
@@ -71,7 +72,7 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
 			name: "MixedValidAndInvalid",
 			input: &ec2.DescribeVolumesInput{
 				VolumeIds: []*string{
-					aws.String("vol-valid123"),
+					aws.String("vol-0123456789abcdef3"),
 					aws.String("invalid-id"),
 				},
 			},
@@ -81,7 +82,7 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
 		{
 			name: "NilVolumeIdInList",
 			input: &ec2.DescribeVolumesInput{
-				VolumeIds: []*string{nil, aws.String("vol-abc123")},
+				VolumeIds: []*string{nil, aws.String("vol-0123456789abcdef1")},
 			},
 			wantErr: false,
 		},
@@ -100,3 +101,15 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
 		})
 	}
 }
+
+// AWS answers a well-prefixed ID whose suffix is not 8 or 17 lowercase hex
+// characters with InvalidParameterValue, not NotFound.
+func TestValidateDescribeVolumesInput_MalformedSuffix(t *testing.T) {
+	for _, id := range []string{"vol-zzz", "vol-0123456789ABCDEF0", "vol-0123456789abcdef", "vol-"} {
+		err := ValidateDescribeVolumesInput(&ec2.DescribeVolumesInput{VolumeIds: []*string{aws.String(id)}})
+		code, msg, ok := awserrors.ResolveErrorDetail(err)
+		assert.True(t, ok, id)
+		assert.Equal(t, awserrors.ErrorInvalidParameterValue, code, id)
+		assert.Equal(t, "Value ("+id+") for parameter volumes is invalid. Expected: 'vol-...'.", msg, id)
+	}
+}

spinifex/domains/ec2/awsapi/volume/handler_test.go

@@ -96,13 +96,6 @@ func TestModifyVolume_NilNATS(t *testing.T) {
 	assert.Error(t, err)
 }
 
-func TestDescribeVolumes_ValidationErrors(t *testing.T) {
-	_, err := DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{
-		VolumeIds: []*string{aws.String("bad-id")},
-	}, nil, "")
-	assert.EqualError(t, err, awserrors.ErrorInvalidVolumeIDMalformed)
-}
-
 func TestDescribeVolumes_NilNATS(t *testing.T) {
 	_, err := DescribeVolumes(context.Background(), nil, nil, "acct-123")
 	assert.Error(t, err)

spinifex/domains/ec2/guestmetadata/metadata.go

@@ -738,7 +738,7 @@ func (s *IMDSServiceImpl) profileFor(ctx context.Context, eni *eniFacts) (*resol
 	}
 	profile, err := s.iam.ResolveInstanceProfile(ctx, eni.iamAccountID(), inst.iamInstanceProfileArn)
 	if err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return nil, roleMissProfileDeleted, nil // profile deleted; treat as no role
 		}
 		slog.ErrorContext(ctx, "IMDS: resolve instance profile failed", "account_id", eni.iamAccountID(), "arn", inst.iamInstanceProfileArn, "err", err)

spinifex/domains/ec2/guestmetadata/metadata_test.go

@@ -1152,7 +1152,7 @@ func TestHTTP_SecurityCredentialsListBackendError500(t *testing.T) {
 // genuine backend error (TestHTTP_*BackendError500) is a 500.
 func TestHTTP_IAMInfoDeletedProfile404(t *testing.T) {
 	res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/gone"}}
-	svc, _ := newTestService(res, &fakeIAM{profileErr: errors.New(awserrors.ErrorIAMNoSuchEntity)}, &fakeAssumer{})
+	svc, _ := newTestService(res, &fakeIAM{profileErr: awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile gone cannot be found.")}, &fakeAssumer{})
 	h := withTapENI(svc.httpHandler(), testENI())
 	token := issueToken(t, h)
 	rec := get(t, h, prefixMetaData+"iam/info", token)

spinifex/domains/ec2/vpc/eni.go

@@ -224,7 +224,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2.
 	groupNames := make(map[string]string)
 	s.fillSGNames(ctx, accountID, record.SecurityGroupIds, groupNames)
 	return &ec2.CreateNetworkInterfaceOutput{
-		NetworkInterface: s.eniRecordToEC2(&record, accountID, groupNames),
+		NetworkInterface: s.eniRecordToEC2(&record, accountID, groupNames, s.vpcDNSHostnames(ctx, accountID, record.VpcId)),
 	}, nil
 }
 
@@ -595,7 +595,7 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *e
 		}
 
 		s.fillSGNames(ctx, accountID, record.SecurityGroupIds, groupNames)
-		enis = append(enis, s.eniRecordToEC2(&record, accountID, groupNames))
+		enis = append(enis, s.eniRecordToEC2(&record, accountID, groupNames, s.vpcDNSHostnames(ctx, accountID, record.VpcId)))
 	}
 
 	// If specific ENI IDs were requested but not found, return error
@@ -1076,9 +1076,26 @@ func (s *VPCServiceImpl) fillSGNames(ctx context.Context, accountID string, ids
 	}
 }
 
+// vpcDNSHostnames reports whether the VPC has enableDnsHostnames on. An
+// unreadable VPC reads as off, so its interfaces go unnamed.
+func (s *VPCServiceImpl) vpcDNSHostnames(ctx context.Context, accountID, vpcID string) bool {
+	entry, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID))
+	if err != nil {
+		slog.WarnContext(ctx, "vpcDNSHostnames: VPC read failed", "vpcId", vpcID, "err", err)
+		return false
+	}
+	var rec VPCRecord
+	if err := json.Unmarshal(entry.Value(), &rec); err != nil {
+		slog.WarnContext(ctx, "vpcDNSHostnames: VPC unmarshal failed", "vpcId", vpcID, "err", err)
+		return false
+	}
+	return rec.EnableDnsHostnames
+}
+
 // eniRecordToEC2 converts an ENI record to an EC2 NetworkInterface, naming
-// each security group from groupNames.
-func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, groupNames map[string]string) *ec2.NetworkInterface {
+// each security group from groupNames. AWS names the interface only while its
+// VPC has enableDnsHostnames on.
+func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, groupNames map[string]string, dnsHostnames bool) *ec2.NetworkInterface {
 	// ENIs with spinifex:managed-by tag are system-managed (e.g. by ELBv2)
 	requesterManaged := record.Tags["spinifex:managed-by"] != ""
 
@@ -1116,7 +1133,7 @@ func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, gro
 		eni.Groups = groups
 	}
 
-	if s.privateDNSName != nil && record.PrivateIpAddress != "" {
+	if dnsHostnames && s.privateDNSName != nil && record.PrivateIpAddress != "" {
 		if name := s.privateDNSName(record.PrivateIpAddress); name != "" {
 			eni.PrivateDnsName = aws.String(name)
 			eni.PrivateIpAddresses[0].PrivateDnsName = aws.String(name)

spinifex/domains/ec2/vpc/eni_test.go

@@ -55,15 +55,11 @@ func TestNetworkInterface_NamesGroupsAndPrivateAddress(t *testing.T) {
 	vpcId := createTestVPC(t, svc, "10.0.0.0/16")
 	subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
 	sgId := createTestSG(t, svc, vpcId, "web")
+	setHostnames(t, svc, vpcId, true)
 
 	// Without a namer an interface carries no private name.
 	unnamed := createTestENI(t, svc, subnetId)
-	desc, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
-		NetworkInterfaceIds: []*string{aws.String(unnamed)},
-	}, testAccountID)
-	require.NoError(t, err)
-	require.Len(t, desc.NetworkInterfaces, 1)
-	assert.Nil(t, desc.NetworkInterfaces[0].PrivateDnsName)
+	assert.Nil(t, describeTestENI(t, svc, unnamed).PrivateDnsName)
 
 	svc.SetPrivateDNSNamer(func(ip string) string { return "name-of-" + ip })
 	out, err := svc.CreateNetworkInterface(context.Background(), &ec2.CreateNetworkInterfaceInput{
@@ -72,13 +68,8 @@ func TestNetworkInterface_NamesGroupsAndPrivateAddress(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 
-	desc, err = svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
-		NetworkInterfaceIds: []*string{out.NetworkInterface.NetworkInterfaceId},
-	}, testAccountID)
-	require.NoError(t, err)
-	require.Len(t, desc.NetworkInterfaces, 1)
-
-	for name, eni := range map[string]*ec2.NetworkInterface{"create": out.NetworkInterface, "describe": desc.NetworkInterfaces[0]} {
+	desc := describeTestENI(t, svc, *out.NetworkInterface.NetworkInterfaceId)
+	for name, eni := range map[string]*ec2.NetworkInterface{"create": out.NetworkInterface, "describe": desc} {
 		want := "name-of-" + aws.StringValue(eni.PrivateIpAddress)
 		assert.Equal(t, want, aws.StringValue(eni.PrivateDnsName), name)
 		require.Len(t, eni.PrivateIpAddresses, 1, name)
@@ -89,6 +80,58 @@ func TestNetworkInterface_NamesGroupsAndPrivateAddress(t *testing.T) {
 	}
 }
 
+func TestNetworkInterface_PrivateDnsNameFollowsVpcDnsHostnames(t *testing.T) {
+	t.Parallel()
+	svc := setupTestVPCService(t)
+	svc.SetPrivateDNSNamer(func(ip string) string { return "name-of-" + ip })
+	vpcId := createTestVPC(t, svc, "10.0.0.0/16")
+	subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
+
+	// A new VPC has enableDnsHostnames off, so its interfaces go unnamed.
+	out, err := svc.CreateNetworkInterface(context.Background(), &ec2.CreateNetworkInterfaceInput{
+		SubnetId: aws.String(subnetId),
+	}, testAccountID)
+	require.NoError(t, err)
+	eniId := *out.NetworkInterface.NetworkInterfaceId
+	for name, eni := range map[string]*ec2.NetworkInterface{"create": out.NetworkInterface, "describe": describeTestENI(t, svc, eniId)} {
+		assert.Nil(t, eni.PrivateDnsName, name)
+		require.Len(t, eni.PrivateIpAddresses, 1, name)
+		assert.Nil(t, eni.PrivateIpAddresses[0].PrivateDnsName, name)
+	}
+
+	// Turning the flag on names the existing interface.
+	setHostnames(t, svc, vpcId, true)
+	eni := describeTestENI(t, svc, eniId)
+	want := "name-of-" + aws.StringValue(eni.PrivateIpAddress)
+	assert.Equal(t, want, aws.StringValue(eni.PrivateDnsName))
+	assert.Equal(t, want, aws.StringValue(eni.PrivateIpAddresses[0].PrivateDnsName))
+
+	// And turning it off again drops the name.
+	setHostnames(t, svc, vpcId, false)
+	eni = describeTestENI(t, svc, eniId)
+	assert.Nil(t, eni.PrivateDnsName)
+	assert.Nil(t, eni.PrivateIpAddresses[0].PrivateDnsName)
+}
+
+func setHostnames(t *testing.T, svc *VPCServiceImpl, vpcId string, on bool) {
+	t.Helper()
+	_, err := svc.ModifyVpcAttribute(context.Background(), &ec2.ModifyVpcAttributeInput{
+		VpcId:              aws.String(vpcId),
+		EnableDnsHostnames: &ec2.AttributeBooleanValue{Value: aws.Bool(on)},
+	}, testAccountID)
+	require.NoError(t, err)
+}
+
+func describeTestENI(t *testing.T, svc *VPCServiceImpl, eniId string) *ec2.NetworkInterface {
+	t.Helper()
+	desc, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
+		NetworkInterfaceIds: []*string{aws.String(eniId)},
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, desc.NetworkInterfaces, 1)
+	return desc.NetworkInterfaces[0]
+}
+
 func TestCreateNetworkInterface_SequentialIPs(t *testing.T) {
 	t.Parallel()
 	svc := setupTestVPCService(t)

spinifex/domains/ecs/records.go

@@ -272,11 +272,15 @@ type TaskRecord struct {
 	// Group / StartedBy mirror the AWS task fields. A service's tasks carry
 	// Group="service:{name}" so the reconciler counts them and the task-state
 	// hook resolves a RUNNING/STOPPED task back to its owning service.
-	Group                string `json:"group,omitempty"`
-	StartedBy            string `json:"startedBy,omitempty"`
-	TaskDefFamily        string `json:"taskDefFamily"`
-	TaskDefRevision      int    `json:"taskDefRevision"`
-	TaskDefARN           string `json:"taskDefArn"`
+	Group           string `json:"group,omitempty"`
+	StartedBy       string `json:"startedBy,omitempty"`
+	TaskDefFamily   string `json:"taskDefFamily"`
+	TaskDefRevision int    `json:"taskDefRevision"`
+	TaskDefARN      string `json:"taskDefArn"`
+	// CredID is the secret under which the task's containers fetch task-role
+	// credentials from the agent. Set only when the task has a role; it must
+	// never be projected into an API response or logged.
+	CredID               string `json:"credId,omitempty"`
 	ContainerInstanceID  string `json:"containerInstanceId,omitempty"`
 	ContainerInstanceARN string `json:"containerInstanceArn,omitempty"`
 	DesiredStatus        string `json:"desiredStatus"`

spinifex/domains/ecs/service.go

@@ -288,7 +288,7 @@ func (s *Service) DescribeClusters(ctx context.Context, input *ecs.DescribeClust
 		names = []string{defaultCluster}
 	}
 	includeSettings := slices.Contains(awsStringSlice(input.Include), ecs.ClusterFieldSettings)
-	out := &ecs.DescribeClustersOutput{}
+	out := &ecs.DescribeClustersOutput{Clusters: []*ecs.Cluster{}, Failures: []*ecs.Failure{}}
 	for _, name := range names {
 		name = ClusterShortName(name)
 		var rec ClusterRecord
@@ -458,7 +458,7 @@ func (s *Service) RegisterTaskDefinition(ctx context.Context, input *ecs.Registe
 	if family == "" {
 		return nil, errors.New(awserrors.ErrorECSInvalidParameter)
 	}
-	if err := validateContainerDefs(input.ContainerDefinitions); err != nil {
+	if err := validateContainerDefs(input.ContainerDefinitions, aws.StringValue(input.Memory)); err != nil {
 		return nil, err
 	}
 	warnUnsupportedLogDrivers(ctx, family, input.ContainerDefinitions)
@@ -497,12 +497,23 @@ func (s *Service) RegisterTaskDefinition(ctx context.Context, input *ecs.Registe
 // The test here is per value, not per field: an empty list or a false/unset
 // flag requests nothing and is accepted, because there is nothing to enforce
 // or refuse. Only a value that asks for the restriction is rejected.
-func validateContainerDefs(defs []*ecs.ContainerDefinition) error {
+//
+// Ahead of those, AWS's own rules: every container needs an image, and without
+// a task-level memory every container needs memory or memoryReservation.
+func validateContainerDefs(defs []*ecs.ContainerDefinition, taskMemory string) error {
 	for _, c := range defs {
 		if c == nil {
 			continue
 		}
 		name := aws.StringValue(c.Name)
+		if aws.StringValue(c.Image) == "" {
+			return awserrors.Errorf(awserrors.ErrorECSClientException,
+				"Container.image should not be null or empty.")
+		}
+		if taskMemory == "" && c.Memory == nil && c.MemoryReservation == nil {
+			return awserrors.Errorf(awserrors.ErrorECSClientException,
+				"Invalid setting for container '%s'. At least one of 'memory' or 'memoryReservation' must be specified.", name)
+		}
 		if len(c.Secrets) > 0 {
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}

spinifex/domains/ecs/service_test.go

@@ -2,7 +2,9 @@ package ecs
 
 import (
 	"context"
+	"encoding/json"
 	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
+	"maps"
 	"strconv"
 	"testing"
 	"time"
@@ -302,6 +304,89 @@ func TestService_RunTask_AssignCarriesTaskRole(t *testing.T) {
 	assert.Equal(t, roleARN, poll.Assignments[0].TaskRoleARN)
 }
 
+// A role task's credential ID is a scheduler-minted secret: not the public task
+// ID, distinct per task, stable on re-delivery, persisted on the task record and
+// absent from every API response. A role-less task gets none.
+func TestService_TaskCredIDIsMintedSecret(t *testing.T) {
+	ctx := context.Background()
+	svc, _ := newTestService(t)
+	_, err := svc.CreateCluster(ctx, &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
+	require.NoError(t, err)
+	_, err = svc.RegisterTaskDefinition(ctx, &ecs.RegisterTaskDefinitionInput{
+		Family:      aws.String("victim"),
+		TaskRoleArn: aws.String("arn:aws:iam::123456789012:role/task-high"),
+		ContainerDefinitions: []*ecs.ContainerDefinition{{
+			Name: aws.String("app"), Image: aws.String("registry/app:1"),
+			Cpu: aws.Int64(128), Memory: aws.Int64(256), Essential: aws.Bool(true),
+		}},
+	}, testAccountID)
+	require.NoError(t, err)
+	registerTaskDef(t, svc, "attacker", 128, 256)
+	registerInstance(t, svc, "web", "i-1", 1024, 2048)
+
+	run, err := svc.RunTask(ctx, &ecs.RunTaskInput{Cluster: aws.String("web"), TaskDefinition: aws.String("victim")}, testAccountID)
+	require.NoError(t, err)
+	start, err := svc.StartTask(ctx, &ecs.StartTaskInput{
+		Cluster: aws.String("web"), TaskDefinition: aws.String("victim"), ContainerInstances: []*string{aws.String("i-1")},
+	}, testAccountID)
+	require.NoError(t, err)
+	_, err = svc.StartTask(ctx, &ecs.StartTaskInput{
+		Cluster: aws.String("web"), TaskDefinition: aws.String("attacker"), ContainerInstances: []*string{aws.String("i-1")},
+	}, testAccountID)
+	require.NoError(t, err)
+
+	poll := func() map[string]ecsv1.Assign {
+		p, perr := svc.PollAssignments(ctx, &PollAssignmentsInput{Cluster: "web", ContainerInstance: "i-1"}, testAccountID)
+		require.NoError(t, perr)
+		byTask := map[string]ecsv1.Assign{}
+		for _, as := range p.Assignments {
+			byTask[as.TaskID] = as
+		}
+		return byTask
+	}
+	first, second := poll(), poll()
+	require.Len(t, first, 3)
+
+	kv, err := svc.bucket(ctx, testAccountID)
+	require.NoError(t, err)
+	var apiJSON []byte
+	for _, out := range []any{run, start} {
+		b, merr := json.Marshal(out)
+		require.NoError(t, merr)
+		apiJSON = append(apiJSON, b...)
+	}
+	var taskARNs []*string
+	for as := range maps.Values(first) {
+		taskARNs = append(taskARNs, aws.String(as.TaskARN))
+	}
+	desc, err := svc.DescribeTasks(ctx, &ecs.DescribeTasksInput{Cluster: aws.String("web"), Tasks: taskARNs}, testAccountID)
+	require.NoError(t, err)
+	b, err := json.Marshal(desc)
+	require.NoError(t, err)
+	apiJSON = append(apiJSON, b...)
+
+	seen := map[string]bool{}
+	for taskID, as := range first {
+		if as.TaskRoleARN == "" {
+			assert.Empty(t, as.CredID, "role-less task got a credential ID")
+			continue
+		}
+		require.NotEmpty(t, as.CredID)
+		assert.NotEqual(t, taskID, as.CredID)
+		assert.False(t, seen[as.CredID], "credential ID reused across tasks")
+		seen[as.CredID] = true
+		assert.Equal(t, as.CredID, second[taskID].CredID, "re-delivered assign changed credential ID")
+
+		var rec TaskRecord
+		found, gerr := getJSON(ctx, kv, TaskKey("web", taskID), &rec)
+		require.NoError(t, gerr)
+		require.True(t, found)
+		assert.Equal(t, as.CredID, rec.CredID)
+		assert.NotContains(t, string(apiJSON), as.CredID, "credential ID leaked in an API response")
+	}
+	assert.Len(t, seen, 2)
+}
+
 // PollAssignments is at-least-once: re-poll without ack re-delivers the assign;
 // acking it (then STOPPED) drains the inbox so it is never re-delivered.
 func TestService_PollAssignments_AckAndReclaim(t *testing.T) {
@@ -812,6 +897,12 @@ func TestService_ListTasks_EmptyIsPresentNotAbsent(t *testing.T) {
 	assert.Empty(t, out.TaskArns)
 }
 
+func TestService_ListTasks_ClusterNotFound(t *testing.T) {
+	svc, _ := newTestService(t)
+	_, err := svc.ListTasks(context.Background(), &ecs.ListTasksInput{Cluster: aws.String("ghost")}, testAccountID)
+	require.EqualError(t, err, awserrors.ErrorECSClusterNotFound)
+}
+
 func TestService_ListContainerInstances_EmptyIsPresentNotAbsent(t *testing.T) {
 	svc, _ := newTestService(t)
 	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
@@ -829,3 +920,21 @@ func TestAccountIDFromBucket(t *testing.T) {
 	_, ok = accountIDFromBucket("some-other-bucket")
 	assert.False(t, ok)
 }
+
+func TestService_DescribeClusters_ListsAlwaysPresent(t *testing.T) {
+	svc, _ := newTestService(t)
+	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
+	require.NoError(t, err)
+
+	found, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{Clusters: []*string{aws.String("web")}}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, found.Clusters, 1)
+	assert.NotNil(t, found.Failures)
+	assert.Empty(t, found.Failures)
+
+	miss, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{Clusters: []*string{aws.String("nope")}}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, miss.Failures, 1)
+	assert.NotNil(t, miss.Clusters)
+	assert.Empty(t, miss.Clusters)
+}

spinifex/domains/ecs/services_test.go

@@ -313,7 +313,7 @@ func TestService_CreateService_ProjectsNetworkConfigurationAndManagedTags(t *tes
 		Family:      aws.String("app"),
 		NetworkMode: aws.String(NetworkModeAwsvpc),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -352,7 +352,7 @@ func TestService_UpdateService_ChangesNetworkConfiguration(t *testing.T) {
 		Family:      aws.String("app"),
 		NetworkMode: aws.String(NetworkModeAwsvpc),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)

spinifex/domains/ecs/task_accounting_test.go

@@ -16,6 +16,8 @@ import (
 // agent to reach anything other than PENDING.
 func seedTasks(t *testing.T, svc *ecsdomain.Service, cluster string, recs ...*ecsdomain.TaskRecord) {
 	t.Helper()
+	_, err := svc.CreateCluster(t.Context(), &ecs.CreateClusterInput{ClusterName: aws.String(cluster)}, ecsdomain.TestAccountID)
+	require.NoError(t, err)
 	kv, err := svc.Bucket(t.Context(), ecsdomain.TestAccountID)
 	require.NoError(t, err)
 	for _, r := range recs {

spinifex/domains/ecs/taskdef_validation_test.go

@@ -19,7 +19,7 @@ func TestRegisterTaskDefinition_RejectsSecrets(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			Secrets: []*ecs.Secret{{
 				Name: aws.String("DB_PASSWORD"), ValueFrom: aws.String("arn:aws:ssm:::parameter/db"),
 			}},
@@ -37,7 +37,7 @@ func TestRegisterTaskDefinition_AcceptsUnsupportedLogDriver(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LogConfiguration: &ecs.LogConfiguration{
 				LogDriver: aws.String("awslogs"),
 				Options:   map[string]*string{"awslogs-group": aws.String("/ecs/app")},
@@ -67,7 +67,7 @@ func TestRegisterTaskDefinition_EchoesRequiresCompatibilities(t *testing.T) {
 		Family:                  aws.String("app"),
 		RequiresCompatibilities: aws.StringSlice([]string{"EC2"}),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -91,7 +91,7 @@ func TestRegisterTaskDefinition_EchoesRuntimePlatform(t *testing.T) {
 			OperatingSystemFamily: aws.String("LINUX"),
 		},
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -113,7 +113,7 @@ func TestRegisterTaskDefinition_RuntimePlatformOmittedWhenUnset(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -253,7 +253,7 @@ func TestRegisterTaskDefinition_EchoesRuntimeFields(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			User:                   aws.String("1000:1000"),
 			ReadonlyRootFilesystem: aws.Bool(true),
 			Privileged:             aws.Bool(false),
@@ -310,7 +310,7 @@ func TestRegisterTaskDefinition_RuntimeFieldsOmittedWhenUnset(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -347,7 +347,7 @@ func TestRegisterTaskDefinition_EchoesEmptyCollections(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			Environment:     []*ecs.KeyValuePair{},
 			MountPoints:     []*ecs.MountPoint{},
 			VolumesFrom:     []*ecs.VolumeFrom{},
@@ -386,7 +386,7 @@ func TestRegisterTaskDefinition_FillsAWSDefaults(t *testing.T) {
 		Family:      aws.String("app"),
 		NetworkMode: aws.String("awsvpc"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			PortMappings: []*ecs.PortMapping{{ContainerPort: aws.Int64(80)}},
 		}},
 	}, testAccountID)
@@ -420,7 +420,7 @@ func TestRegisterTaskDefinition_BridgeLeavesHostPortUnset(t *testing.T) {
 		Family:      aws.String("app"),
 		NetworkMode: aws.String("bridge"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			PortMappings: []*ecs.PortMapping{{ContainerPort: aws.Int64(53), Protocol: aws.String("udp")}},
 		}},
 	}, testAccountID)
@@ -435,7 +435,7 @@ func TestRegisterTaskDefinition_AcceptsEmptyRuntimeFields(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			MountPoints:    []*ecs.MountPoint{},
 			VolumesFrom:    []*ecs.VolumeFrom{},
 			SystemControls: []*ecs.SystemControl{},
@@ -455,7 +455,7 @@ func TestRegisterTaskDefinition_RejectsMountPoints(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			MountPoints: []*ecs.MountPoint{{
 				SourceVolume: aws.String("data"), ContainerPath: aws.String("/data"),
 			}},
@@ -476,7 +476,7 @@ func TestRegisterTaskDefinition_RejectsVolumesFrom(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			VolumesFrom: []*ecs.VolumeFrom{{SourceContainer: aws.String("other")}},
 		}},
 	}, testAccountID)
@@ -494,7 +494,7 @@ func TestRegisterTaskDefinition_RejectsLinuxParametersDevices(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LinuxParameters: &ecs.LinuxParameters{
 				Devices: []*ecs.Device{{HostPath: aws.String("/dev/fuse")}},
 			},
@@ -514,7 +514,7 @@ func TestRegisterTaskDefinition_RejectsInitProcessEnabled(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LinuxParameters: &ecs.LinuxParameters{InitProcessEnabled: aws.Bool(true)},
 		}},
 	}, testAccountID)
@@ -533,7 +533,7 @@ func TestRegisterTaskDefinition_RejectsSharedMemoryAndTmpfs(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("shm"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LinuxParameters: &ecs.LinuxParameters{SharedMemorySize: aws.Int64(64)},
 		}},
 	}, testAccountID)
@@ -545,7 +545,7 @@ func TestRegisterTaskDefinition_RejectsSharedMemoryAndTmpfs(t *testing.T) {
 	_, err = svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("tmpfs"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LinuxParameters: &ecs.LinuxParameters{
 				Tmpfs: []*ecs.Tmpfs{{ContainerPath: aws.String("/tmp"), Size: aws.Int64(64)}},
 			},
@@ -567,7 +567,7 @@ func TestRunTask_AssignCarriesRuntimeFields(t *testing.T) {
 	_, err = svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			User:                   aws.String("1000"),
 			ReadonlyRootFilesystem: aws.Bool(true),
 			Privileged:             aws.Bool(true),
@@ -608,3 +608,42 @@ func TestRunTask_AssignCarriesRuntimeFields(t *testing.T) {
 	assert.Equal(t, "net.core.somaxconn", ac.SystemControls[0].Namespace)
 	assert.Equal(t, []string{"SYS_PTRACE"}, ac.CapAdd)
 }
+
+// AWS rejects a container with no image before checking its memory, and
+// without task-level memory requires memory or memoryReservation per container.
+func TestRegisterTaskDefinition_RequiresImageAndMemory(t *testing.T) {
+	tests := []struct {
+		name       string
+		taskMemory *string
+		container  *ecs.ContainerDefinition
+		wantErr    string
+	}{
+		{"no image or memory", nil, &ecs.ContainerDefinition{Name: aws.String("web")},
+			"Container.image should not be null or empty."},
+		{"empty image", aws.String("512"), &ecs.ContainerDefinition{Name: aws.String("web"), Image: aws.String("")},
+			"Container.image should not be null or empty."},
+		{"no memory", nil, &ecs.ContainerDefinition{Name: aws.String("web"), Image: aws.String("registry/web:1")},
+			"Invalid setting for container 'web'. At least one of 'memory' or 'memoryReservation' must be specified."},
+		{"memoryReservation only", nil, &ecs.ContainerDefinition{
+			Name: aws.String("web"), Image: aws.String("registry/web:1"), MemoryReservation: aws.Int64(128)}, ""},
+		{"task-level memory", aws.String("512"), &ecs.ContainerDefinition{
+			Name: aws.String("web"), Image: aws.String("registry/web:1")}, ""},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			svc, _ := newTestService(t)
+			_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
+				Family: aws.String("app"), Memory: tc.taskMemory,
+				ContainerDefinitions: []*ecs.ContainerDefinition{tc.container},
+			}, testAccountID)
+			if tc.wantErr == "" {
+				require.NoError(t, err)
+				return
+			}
+			code, msg, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok)
+			assert.Equal(t, awserrors.ErrorECSClientException, code)
+			assert.Equal(t, tc.wantErr, msg)
+		})
+	}
+}

spinifex/domains/ecs/tasks.go

@@ -237,6 +237,9 @@ func (s *Service) newTaskRecord(accountID, cluster, taskID string, td *taskdefin
 		GPU:                  gpu,
 		CreatedAt:            now,
 	}
+	if td.TaskRoleArn != "" {
+		rec.CredID = uuid.NewV4().String()
+	}
 	for _, c := range td.Containers {
 		rec.Containers = append(rec.Containers, ContainerState{Name: c.Name, Status: TaskStatusPending})
 	}
@@ -257,6 +260,7 @@ func (s *Service) publishAssign(ctx context.Context, kv jetstream.KeyValue, acco
 		TaskDefFamily:    td.Family,
 		TaskDefRevision:  td.Revision,
 		TaskRoleARN:      td.TaskRoleArn,
+		CredID:           rec.CredID,
 		ExecutionRoleARN: td.ExecutionRoleArn,
 		ENIID:            rec.ENIID,
 		ENIMacAddress:    rec.ENIMacAddress,
@@ -303,6 +307,14 @@ func (s *Service) ListTasks(ctx context.Context, input *ecs.ListTasksInput, acco
 	if err != nil {
 		return nil, err
 	}
+	var clusterRec ClusterRecord
+	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
+	if err != nil {
+		return nil, err
+	}
+	if !found {
+		return nil, errors.New(awserrors.ErrorECSClusterNotFound)
+	}
 	keys, err := keysWithPrefix(ctx, kv, TasksPrefix(cluster))
 	if err != nil {
 		return nil, err

spinifex/domains/eks/addon/catalog.go

@@ -4,8 +4,14 @@ import (
 	"fmt"
 	"maps"
 	"slices"
+	"strconv"
+	"strings"
 )
 
+// firstBuild is the EKS build number of every bundled version; it moves per
+// version when Spinifex rebuilds a bundle.
+const firstBuild = 1
+
 // NvidiaDevicePlugin is staged by a GPU node group's launch (EnsureGPUDevicePlugin),
 // never user-requested directly.
 const NvidiaDevicePlugin = "nvidia-device-plugin"
@@ -18,6 +24,11 @@ type Spec struct {
 	// Versions lists supported versions newest-first; [0] is the default.
 	Versions       []string
 	DefaultVersion string
+	// Builds maps each upstream version to its EKS build number, the n in
+	// v<upstream>-eksbuild.<n>.
+	Builds map[string]int
+	// Type is the AWS add-on type reported by DescribeAddonVersions.
+	Type string
 	// RequiresIRSA indicates the add-on needs an IAM role for service-account binding.
 	RequiresIRSA bool
 	Description  string
@@ -28,26 +39,26 @@ type Spec struct {
 
 // catalog is the bundled add-on registry. Keep newest version first per slice.
 var catalog = buildCatalog(
-	newSpec("aws-load-balancer-controller", true,
+	typed("networking", newSpec("aws-load-balancer-controller", true,
 		"Provisions ELBv2 load balancers for Kubernetes Service/Ingress resources.",
-		"2.11.0"),
-	newSpec("argocd", false,
+		"2.11.0")),
+	typed("gitops", newSpec("argocd", false,
 		"Declarative GitOps continuous delivery for Kubernetes.",
-		"3.0.23"),
-	newSpec("aws-ebs-csi-driver", true,
+		"3.0.23")),
+	typed("storage", newSpec("aws-ebs-csi-driver", true,
 		"Container Storage Interface driver for Amazon EBS (Viperblock) volumes.",
-		"1.40.1"),
+		"1.40.1")),
 	// Staged automatically for GPU node groups, so hidden from the public
 	// catalogue like spinifex-noop but still creatable by name.
-	hidden(newSpec(NvidiaDevicePlugin, false,
+	hidden(typed("accelerator", newSpec(NvidiaDevicePlugin, false,
 		"NVIDIA device plugin exposing nvidia.com/gpu allocatable via CDI on GPU-tainted nodes.",
-		"0.17.4")),
+		"0.17.4"))),
 	// spinifex-noop is the delivery-transport fixture: a trivial bundle
 	// (Namespace + ConfigMap) the add-on e2e uses to prove stage, render,
 	// auto-deploy, ACTIVE and delete round-trip without a real workload.
-	hidden(newSpec("spinifex-noop", false,
+	hidden(typed("fixture", newSpec("spinifex-noop", false,
 		"No-op delivery-transport fixture (Namespace + ConfigMap).",
-		"0.1.0")),
+		"0.1.0"))),
 )
 
 // hidden marks a spec as internal: still creatable, but absent from the
@@ -57,6 +68,12 @@ func hidden(s Spec) Spec {
 	return s
 }
 
+// typed sets the AWS add-on type reported by DescribeAddonVersions.
+func typed(addonType string, s Spec) Spec {
+	s.Type = addonType
+	return s
+}
+
 // newSpec builds a spec with the first version as its default.
 func newSpec(name string, requiresIRSA bool, description string, versions ...string) Spec {
 	spec := Spec{
@@ -68,6 +85,10 @@ func newSpec(name string, requiresIRSA bool, description string, versions ...str
 	if len(versions) > 0 {
 		spec.DefaultVersion = versions[0]
 	}
+	spec.Builds = make(map[string]int, len(versions))
+	for _, v := range versions {
+		spec.Builds[v] = firstBuild
+	}
 	return spec
 }
 
@@ -111,6 +132,50 @@ func (s Spec) SupportsVersion(version string) bool {
 	return slices.Contains(s.Versions, version)
 }
 
+// awsForm renders a catalog version as v<upstream>-eksbuild.<build>.
+func (s Spec) awsForm(upstream string) string {
+	build, ok := s.Builds[upstream]
+	if !ok {
+		build = firstBuild
+	}
+	return "v" + upstream + "-eksbuild." + strconv.Itoa(build)
+}
+
+// AWSVersion returns the AWS form of a stored version. A version the catalogue
+// does not list, or one already in AWS form, is returned unchanged.
+func (s Spec) AWSVersion(stored string) string {
+	if !slices.Contains(s.Versions, stored) {
+		return stored
+	}
+	return s.awsForm(stored)
+}
+
+// AWSVersions returns every supported version in AWS form, newest first.
+func (s Spec) AWSVersions() []string {
+	out := make([]string, len(s.Versions))
+	for i, v := range s.Versions {
+		out[i] = s.awsForm(v)
+	}
+	return out
+}
+
+// ParseVersion maps a client-supplied version, in AWS form or bare upstream
+// form, to the upstream version stored on the record. Nothing else is accepted.
+func (s Spec) ParseVersion(input string) (string, bool) {
+	if slices.Contains(s.Versions, input) {
+		return input, true
+	}
+	if !strings.HasPrefix(input, "v") {
+		return "", false
+	}
+	for _, v := range s.Versions {
+		if input == s.awsForm(v) {
+			return v, true
+		}
+	}
+	return "", false
+}
+
 // Specs returns every catalogue entry, hidden ones included, sorted by name.
 func Specs() []Spec {
 	names := slices.Sorted(maps.Keys(catalog))

spinifex/domains/eks/addon/catalog_test.go

@@ -100,3 +100,78 @@ func TestValidateCatalog_ChecksTheBundledCatalog(t *testing.T) {
 	t.Cleanup(func() { catalog = previous })
 	require.EqualError(t, ValidateCatalog(), `add-on "broken" has no versions`)
 }
+
+// buildSpec is a two-version spec whose newest version is on EKS build 2, so
+// a test cannot pass by hardcoding build 1.
+func buildSpec() Spec {
+	return Spec{
+		Name:     "fixture",
+		Versions: []string{"2.0.0", "1.0.0"},
+		Builds:   map[string]int{"2.0.0": 2, "1.0.0": 1},
+	}
+}
+
+func TestSpec_AWSVersionUsesTheCatalogBuildNumber(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	assert.Equal(t, "v2.0.0-eksbuild.2", spec.AWSVersion("2.0.0"))
+	assert.Equal(t, "v1.0.0-eksbuild.1", spec.AWSVersion("1.0.0"))
+}
+
+func TestSpec_AWSVersionIsIdempotent(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	assert.Equal(t, "v2.0.0-eksbuild.2", spec.AWSVersion("v2.0.0-eksbuild.2"),
+		"a record already holding the AWS form must not be wrapped twice")
+}
+
+func TestSpec_AWSVersionsAreNewestFirst(t *testing.T) {
+	t.Parallel()
+	assert.Equal(t, []string{"v2.0.0-eksbuild.2", "v1.0.0-eksbuild.1"}, buildSpec().AWSVersions())
+}
+
+func TestSpec_ParseVersionAcceptsOnlyTheExactForms(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	for input, want := range map[string]string{
+		"v2.0.0-eksbuild.2": "2.0.0",
+		"2.0.0":             "2.0.0",
+		"v1.0.0-eksbuild.1": "1.0.0",
+		"1.0.0":             "1.0.0",
+	} {
+		got, ok := spec.ParseVersion(input)
+		assert.True(t, ok, "%q must be accepted", input)
+		assert.Equal(t, want, got, "%q", input)
+	}
+}
+
+func TestSpec_ParseVersionRefusesEverythingElse(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	for _, input := range []string{
+		"v2.0.0-eksbuild.1", // wrong build for 2.0.0
+		"v2.0.0-eksbuild.3",
+		"v1.0.0-eksbuild.2", // wrong build for 1.0.0
+		"v2.0.0",            // missing build
+		"2.0",               // partial
+		"v2.0.0-eksbuild.",
+		"2.0.0-eksbuild.2", // missing v
+		"V2.0.0-eksbuild.2",
+		" 2.0.0",
+		"9.9.9",
+		"",
+	} {
+		_, ok := spec.ParseVersion(input)
+		assert.False(t, ok, "%q must be refused", input)
+	}
+}
+
+func TestCatalog_EveryVersionHasAnExplicitBuild(t *testing.T) {
+	t.Parallel()
+	for _, spec := range Specs() {
+		for _, v := range spec.Versions {
+			assert.Equal(t, 1, spec.Builds[v], "%s %s needs an explicit build number", spec.Name, v)
+			assert.Equal(t, "v"+v+"-eksbuild.1", spec.AWSVersion(v), "%s %s", spec.Name, v)
+		}
+	}
+}

spinifex/domains/eks/awsapi/addon_versions_query_test.go

@@ -0,0 +1,105 @@
+package awsapi
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// captureDescribeAddonVersions answers eks.DescribeAddonVersions and returns a
+// function reading the last input the gateway sent to the service.
+func captureDescribeAddonVersions(t *testing.T, nc *nats.Conn) func() eks.DescribeAddonVersionsInput {
+	t.Helper()
+	var (
+		mu   sync.Mutex
+		last eks.DescribeAddonVersionsInput
+	)
+	sub, err := nc.Subscribe("eks.DescribeAddonVersions", func(m *nats.Msg) {
+		mu.Lock()
+		last = eks.DescribeAddonVersionsInput{}
+		_ = json.Unmarshal(m.Data, &last)
+		mu.Unlock()
+		_ = m.Respond([]byte(`{}`))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	return func() eks.DescribeAddonVersionsInput {
+		mu.Lock()
+		defer mu.Unlock()
+		return last
+	}
+}
+
+// The gateway folds a GET query string into the body as a JSON object of
+// string lists; the model binds every field below from the query.
+func TestDescribeAddonVersions_PassesEveryModelQueryParameter(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	got := captureDescribeAddonVersions(t, nc)
+
+	query := []byte(`{"addonName":["aws-ebs-csi-driver"],"kubernetesVersion":["1.32"],` +
+		`"maxResults":["2"],"nextToken":["tok"],"types":["storage","networking"],` +
+		`"publishers":["eks"],"owners":["aws"]}`)
+	_, err := DescribeAddonVersions(context.Background(), nc, acct, query)
+	require.NoError(t, err)
+
+	in := got()
+	assert.Equal(t, "aws-ebs-csi-driver", aws.StringValue(in.AddonName))
+	assert.Equal(t, "1.32", aws.StringValue(in.KubernetesVersion))
+	assert.Equal(t, int64(2), aws.Int64Value(in.MaxResults))
+	assert.Equal(t, "tok", aws.StringValue(in.NextToken))
+	assert.Equal(t, []string{"storage", "networking"}, aws.StringValueSlice(in.Types))
+	assert.Equal(t, []string{"eks"}, aws.StringValueSlice(in.Publishers))
+	assert.Equal(t, []string{"aws"}, aws.StringValueSlice(in.Owners))
+}
+
+func TestDescribeAddonVersions_NoQueryLeavesEveryFilterUnset(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	got := captureDescribeAddonVersions(t, nc)
+
+	_, err := DescribeAddonVersions(context.Background(), nc, acct, nil)
+	require.NoError(t, err)
+
+	in := got()
+	assert.Nil(t, in.AddonName)
+	assert.Nil(t, in.KubernetesVersion)
+	assert.Nil(t, in.MaxResults)
+	assert.Nil(t, in.NextToken)
+	assert.Empty(t, in.Types)
+}
+
+func TestDescribeAddonVersions_RefusesMalformedQuery(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	captureDescribeAddonVersions(t, nc)
+
+	_, err := DescribeAddonVersions(context.Background(), nc, acct, []byte(`{"maxResults":["many"]}`))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "maxResults")
+
+	_, err = DescribeAddonVersions(context.Background(), nc, acct, []byte(`not json`))
+	require.Error(t, err)
+}
+
+// The route must hand the folded query to the wrapper, not drop it.
+func TestDescribeAddonVersionsRoute_PassesTheQuery(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	got := captureDescribeAddonVersions(t, nc)
+
+	for _, r := range routes {
+		if r.Action != "DescribeAddonVersions" {
+			continue
+		}
+		_, err := r.Handler(context.Background(), nc, acct, "", nil, []byte(`{"addonName":["argocd"]}`))
+		require.NoError(t, err)
+		assert.Equal(t, "argocd", aws.StringValue(got().AddonName))
+		return
+	}
+	t.Fatal("no DescribeAddonVersions route")
+}

spinifex/domains/eks/awsapi/addons.go

@@ -16,8 +16,23 @@ func ListAddons(ctx context.Context, natsConn *nats.Conn, accountID, cluster str
 }
 
 // DescribeAddonVersions — GET /addons/supported-versions.
-func DescribeAddonVersions(ctx context.Context, natsConn *nats.Conn, accountID string) (*eks.DescribeAddonVersionsOutput, error) {
-	return ekscluster.NewNATSEKSService(natsConn).DescribeAddonVersions(ctx, &eks.DescribeAddonVersionsInput{}, accountID)
+func DescribeAddonVersions(ctx context.Context, natsConn *nats.Conn, accountID string, query []byte) (*eks.DescribeAddonVersionsOutput, error) {
+	q, err := parseListQuery(query)
+	if err != nil {
+		return nil, err
+	}
+	input := &eks.DescribeAddonVersionsInput{
+		AddonName:         q.str("addonName"),
+		KubernetesVersion: q.str("kubernetesVersion"),
+		NextToken:         q.str("nextToken"),
+		Types:             q.strs("types"),
+		Publishers:        q.strs("publishers"),
+		Owners:            q.strs("owners"),
+	}
+	if input.MaxResults, err = q.int64("maxResults"); err != nil {
+		return nil, err
+	}
+	return ekscluster.NewNATSEKSService(natsConn).DescribeAddonVersions(ctx, input, accountID)
 }
 
 // CreateAddon — POST /clusters/{name}/addons.

spinifex/domains/eks/awsapi/query.go

@@ -0,0 +1,61 @@
+package awsapi
+
+import (
+	"encoding/json"
+	"strconv"
+
+	"github.com/aws/aws-sdk-go/aws"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// listQuery holds the query-string inputs of a GET action, which the gateway
+// folds into the body as a JSON object of string lists.
+type listQuery map[string][]string
+
+// parseListQuery decodes the folded query of a GET action. An empty body means
+// the request carried no query string.
+func parseListQuery(body []byte) (listQuery, error) {
+	q := listQuery{}
+	if len(body) == 0 {
+		return q, nil
+	}
+	if err := json.Unmarshal(body, (*map[string][]string)(&q)); err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "malformed query string")
+	}
+	return q, nil
+}
+
+// str returns the first value of name, or nil when it is absent or empty.
+func (q listQuery) str(name string) *string {
+	if v := q[name]; len(v) > 0 && v[0] != "" {
+		return aws.String(v[0])
+	}
+	return nil
+}
+
+// strs returns every non-empty value of name, or nil when there are none.
+func (q listQuery) strs(name string) []*string {
+	var out []string
+	for _, v := range q[name] {
+		if v != "" {
+			out = append(out, v)
+		}
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return aws.StringSlice(out)
+}
+
+// int64 returns the first value of name as an integer, or nil when absent.
+func (q listQuery) int64(name string) (*int64, error) {
+	s := q.str(name)
+	if s == nil {
+		return nil, nil
+	}
+	n, err := strconv.ParseInt(*s, 10, 64)
+	if err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "%s must be an integer", name)
+	}
+	return aws.Int64(n), nil
+}

spinifex/domains/eks/awsapi/registration.go

@@ -82,17 +82,39 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	if err == nil {
 		return nil
 	}
-	if action, _, _, ok := router.Lookup(inv.Request.Method, inv.Request.URL.EscapedPath()); ok && IsInternalAction(action) {
+	action, _, _, routed := router.Lookup(inv.Request.Method, inv.Request.URL.EscapedPath())
+	if routed && IsInternalAction(action) {
 		return err
 	}
 	code, msg, ok := awserrors.ResolveErrorDetail(err)
 	if !ok || code != awserrors.ErrorInvalidParameterValue {
 		return err
 	}
-	if msg == "" {
-		return errors.New(awserrors.ErrorEKSInvalidParameter)
+	wire := awserrors.ErrorEKSInvalidParameter
+	if override, ok := parameterRefusalCodes[action]; ok {
+		wire = override
+		if msg == "" {
+			msg = "a parameter of the request is not valid"
+		}
 	}
-	return awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "%s", msg)
+	if msg == "" {
+		return errors.New(wire)
+	}
+	return awserrors.Errorf(wire, "%s", msg)
+}
+
+// parameterRefusalCodes holds the operations whose model does not declare
+// InvalidParameterException, with the declared code a refused parameter uses.
+var parameterRefusalCodes = map[string]string{
+	"DescribeCluster":              awserrors.ErrorEKSClient,
+	"DeleteCluster":                awserrors.ErrorEKSClient,
+	"DescribeAccessEntry":          awserrors.ErrorEKSInvalidRequest,
+	"DeleteAccessEntry":            awserrors.ErrorEKSInvalidRequest,
+	"DisassociateAccessPolicy":     awserrors.ErrorEKSInvalidRequest,
+	"ListAssociatedAccessPolicies": awserrors.ErrorEKSInvalidRequest,
+	"TagResource":                  awserrors.ErrorEKSBadRequest,
+	"UntagResource":                awserrors.ErrorEKSBadRequest,
+	"ListTagsForResource":          awserrors.ErrorEKSBadRequest,
 }
 
 // serve resolves the route, authorizes the request, then runs the action.

spinifex/domains/eks/awsapi/registration_error_codes_test.go

@@ -0,0 +1,98 @@
+//test:in-package — routes is unexported, and the table must cover exactly the
+// actions the registration serves.
+
+package awsapi
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/bluebottle/pkg/iampolicy"
+	"github.com/mulgadc/spinifex/internal/awsmodel"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// routePath fills a route pattern with placeholder segments. The tag routes
+// take an ARN, which has to be escaped to stay one path segment.
+func routePath(pattern string) string {
+	arn := url.PathEscape("arn:aws:eks:us-east-1:111122223333:cluster/c")
+	path := strings.ReplaceAll(pattern, "*", arn)
+	for {
+		start := strings.Index(path, "{")
+		if start < 0 {
+			return path
+		}
+		end := strings.Index(path[start:], "}")
+		path = path[:start] + "x" + path[start+end+1:]
+	}
+}
+
+// A parameter refusal must reach the wire under a code the operation declares
+// in the EKS model, or SDK clients cannot match it to a typed error.
+func TestRegistration_ParameterRefusalCodeIsDeclaredByOperation(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	sub, err := nc.Subscribe("eks.>", func(m *nats.Msg) {
+		if m.Reply != "" {
+			_ = m.Respond(awserrors.GenerateErrorPayload(awserrors.ErrorInvalidParameterValue))
+		}
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	model, err := awsmodel.Load(awsmodel.EKS)
+	require.NoError(t, err)
+	registration := NewRegistration(Deps{NATS: nc})
+
+	for _, route := range routes {
+		if IsInternalAction(route.Action) {
+			continue
+		}
+		// The handler takes no input and the model declares no client error.
+		if route.Action == "ListAccessPolicies" {
+			continue
+		}
+		t.Run(route.Action, func(t *testing.T) {
+			operation, ok := model.Operation(route.Action)
+			require.True(t, ok, "%s is not an operation in the EKS model", route.Action)
+			declared := make([]string, 0, len(operation.Errors))
+			for _, ref := range operation.Errors {
+				declared = append(declared, ref.Shape)
+			}
+
+			req := httptest.NewRequest(route.Method, routePath(route.Pattern), strings.NewReader(`{}`))
+			inv := dispatch.Invocation{
+				Request:   req,
+				AccountID: "111122223333",
+				Region:    "us-east-1",
+				Authorize: func(string, string, []string, iampolicy.ConditionKeys) error { return nil },
+				Actor: func() (dispatch.Actor, error) {
+					return dispatch.Actor{AccountID: "111122223333", PrincipalType: "user", UserName: "alice"}, nil
+				},
+			}
+
+			err := registration.Dispatch(httptest.NewRecorder(), inv)
+
+			require.Error(t, err)
+			code, ok := awserrors.ResolveErrorCode(err)
+			require.True(t, ok, "unresolvable error code: %v", err)
+			assert.Contains(t, declared, code,
+				"%s %s refuses a parameter with %s, which the operation does not declare", route.Method, route.Pattern, code)
+		})
+	}
+}
+
+// The access operations declare InvalidRequestException, not a parameter code.
+func TestEKSInvalidRequestCodeIsRegistered(t *testing.T) {
+	assert.Equal(t, "InvalidRequestException", awserrors.ErrorEKSInvalidRequest)
+	assert.Equal(t, http.StatusBadRequest, awserrors.HTTPStatusForError(awserrors.Errorf(awserrors.ErrorEKSInvalidRequest, "x")))
+	assert.Equal(t, "BadRequestException", awserrors.ErrorEKSBadRequest)
+	assert.Equal(t, http.StatusBadRequest, awserrors.HTTPStatusForError(awserrors.Errorf(awserrors.ErrorEKSBadRequest, "x")))
+}

spinifex/domains/eks/awsapi/routes.go

@@ -131,7 +131,7 @@ var routes = []route{
 	// Addons
 	{Method: "GET", Pattern: "/addons/supported-versions", Action: "DescribeAddonVersions",
 		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {
-			return DescribeAddonVersions(ctx, nc, acct)
+			return DescribeAddonVersions(ctx, nc, acct, b)
 		}},
 	{Method: "GET", Pattern: "/clusters/{clusterName}/addons", Action: "ListAddons",
 		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {

spinifex/domains/eks/awsapi/wrappers_test.go

@@ -176,7 +176,7 @@ func TestGatewayWrappers_Addons(t *testing.T) {
 	require.NoError(t, err)
 	assert.NotNil(t, out1)
 
-	out2, err := DescribeAddonVersions(context.Background(), nc, acct)
+	out2, err := DescribeAddonVersions(context.Background(), nc, acct, nil)
 	require.NoError(t, err)
 	assert.NotNil(t, out2)
 

spinifex/domains/eks/cluster/addon_describe_versions_test.go

@@ -0,0 +1,171 @@
+package cluster
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/domains/eks/addon"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func describeVersions(t *testing.T, svc *EKSServiceImpl, in *eks.DescribeAddonVersionsInput) *eks.DescribeAddonVersionsOutput {
+	t.Helper()
+	out, err := svc.DescribeAddonVersions(context.Background(), in, testAccountID)
+	require.NoError(t, err)
+	return out
+}
+
+// The terraform aws_eks_addon_version data source asks for one add-on at the
+// cluster's Kubernetes version, then takes the version whose compatibility for
+// that cluster version is the default (or, with most_recent, the first one).
+func TestDescribeAddonVersions_TerraformProviderCall(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	out := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{
+		AddonName:         aws.String(csiAddon),
+		KubernetesVersion: aws.String(defaultK8sVersion),
+	})
+	require.Len(t, out.Addons, 1)
+	info := out.Addons[0]
+	assert.Equal(t, csiAddon, aws.StringValue(info.AddonName))
+	assert.Nil(t, out.NextToken)
+
+	spec, ok := addon.Lookup(csiAddon)
+	require.True(t, ok)
+	require.Equal(t, spec.AWSVersions(), versionStrings(info), "newest first, AWS form")
+
+	defaults := 0
+	for _, v := range info.AddonVersions {
+		require.Len(t, v.Compatibilities, 1, "one entry, for the requested cluster version")
+		c := v.Compatibilities[0]
+		assert.Equal(t, defaultK8sVersion, aws.StringValue(c.ClusterVersion))
+		assert.Equal(t, []string{"*"}, aws.StringValueSlice(c.PlatformVersions))
+		if aws.BoolValue(c.DefaultVersion) {
+			defaults++
+			assert.Equal(t, spec.AWSVersion(spec.DefaultVersion), aws.StringValue(v.AddonVersion))
+		}
+	}
+	assert.Equal(t, 1, defaults, "exactly one defaultVersion true")
+}
+
+// AWS observed (EKS-01): compatibilities holds only the requested version. The
+// response for a version no add-on supports is not captured; this pins the
+// shape #61 chose, an empty list with no error.
+func TestDescribeAddonVersions_UnsupportedKubernetesVersionIsEmpty(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	out := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{
+		AddonName:         aws.String(csiAddon),
+		KubernetesVersion: aws.String("1.29"),
+	})
+	assert.Empty(t, out.Addons)
+	assert.Nil(t, out.NextToken)
+}
+
+func TestDescribeAddonVersions_FiltersByNameAndHidesFixturesUnlessNamed(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	for _, name := range []string{csiAddon, "argocd", albController} {
+		out := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{AddonName: aws.String(name)})
+		require.Len(t, out.Addons, 1, name)
+		assert.Equal(t, name, aws.StringValue(out.Addons[0].AddonName))
+	}
+	assert.Empty(t, describeVersions(t, svc, &eks.DescribeAddonVersionsInput{AddonName: aws.String("vpc-cni")}).Addons)
+}
+
+// maxResults counts add-ons, not versions; a page boundary never splits an
+// add-on's version list, and the last page carries no token.
+func TestDescribeAddonVersions_PagesByAddon(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	all := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{})
+	require.Greater(t, len(all.Addons), 1)
+	assert.Nil(t, all.NextToken)
+
+	var names []string
+	in := &eks.DescribeAddonVersionsInput{MaxResults: aws.Int64(1)}
+	for range len(all.Addons) {
+		out := describeVersions(t, svc, in)
+		require.Len(t, out.Addons, 1)
+		names = append(names, aws.StringValue(out.Addons[0].AddonName))
+		assert.Equal(t, versionStrings(all.Addons[len(names)-1]), versionStrings(out.Addons[0]))
+		in.NextToken = out.NextToken
+	}
+	assert.Nil(t, in.NextToken, "last page carries no token")
+	var want []string
+	for _, a := range all.Addons {
+		want = append(want, aws.StringValue(a.AddonName))
+	}
+	assert.Equal(t, want, names)
+}
+
+// Every add-on carries the descriptive fields the data source and console read.
+func TestDescribeAddonVersions_CarriesAddonAndVersionMetadata(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	for _, a := range describeVersions(t, svc, &eks.DescribeAddonVersionsInput{}).Addons {
+		name := aws.StringValue(a.AddonName)
+		assert.NotEmpty(t, aws.StringValue(a.Type), name)
+		assert.NotEmpty(t, aws.StringValue(a.Owner), name)
+		assert.NotEmpty(t, aws.StringValue(a.Publisher), name)
+		assert.Nil(t, a.MarketplaceInformation, name)
+		for _, v := range a.AddonVersions {
+			assert.NotEmpty(t, v.Architecture, name)
+			assert.NotNil(t, v.RequiresConfiguration, name)
+			assert.NotNil(t, v.RequiresIamPermissions, name)
+		}
+	}
+}
+
+// types, publishers and owners are model filters; each keeps add-ons whose
+// field matches one of the given values.
+func TestDescribeAddonVersions_FiltersByTypePublisherOwner(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	first := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{}).Addons[0]
+	cases := []struct {
+		name  string
+		match func(*eks.DescribeAddonVersionsInput, []*string)
+		value *string
+	}{
+		{"types", func(in *eks.DescribeAddonVersionsInput, v []*string) { in.Types = v }, first.Type},
+		{"publishers", func(in *eks.DescribeAddonVersionsInput, v []*string) { in.Publishers = v }, first.Publisher},
+		{"owners", func(in *eks.DescribeAddonVersionsInput, v []*string) { in.Owners = v }, first.Owner},
+	}
+	for _, c := range cases {
+		hit := &eks.DescribeAddonVersionsInput{}
+		c.match(hit, aws.StringSlice([]string{"nobody", aws.StringValue(c.value)}))
+		out := describeVersions(t, svc, hit)
+		require.NotEmpty(t, out.Addons, c.name)
+		assert.Contains(t, addonNames(out), aws.StringValue(first.AddonName), c.name)
+
+		miss := &eks.DescribeAddonVersionsInput{}
+		c.match(miss, aws.StringSlice([]string{"nobody"}))
+		assert.Empty(t, describeVersions(t, svc, miss).Addons, c.name)
+	}
+}
+
+func versionStrings(a *eks.AddonInfo) []string {
+	out := make([]string, 0, len(a.AddonVersions))
+	for _, v := range a.AddonVersions {
+		out = append(out, aws.StringValue(v.AddonVersion))
+	}
+	return out
+}
+
+func addonNames(o *eks.DescribeAddonVersionsOutput) []string {
+	out := make([]string, 0, len(o.Addons))
+	for _, a := range o.Addons {
+		out = append(out, aws.StringValue(a.AddonName))
+	}
+	return out
+}

spinifex/domains/eks/cluster/addon_version_format_test.go

@@ -0,0 +1,220 @@
+package cluster
+
+import (
+	"bytes"
+	"context"
+	"log/slog"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/domains/eks/addon"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	csiAddon       = "aws-ebs-csi-driver"
+	csiBare        = "1.40.1"
+	csiAWSVersion  = "v1.40.1-eksbuild.1"
+	csiWrongBuild  = "v1.40.1-eksbuild.2"
+	versionParam   = "addonVersion"
+	notBeforeAWSOK = "an AWS-format version must be accepted"
+)
+
+func TestDescribeAddonVersions_ReturnsAWSFormNewestFirst(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	out, err := svc.DescribeAddonVersions(context.Background(), &eks.DescribeAddonVersionsInput{}, testAccountID)
+	require.NoError(t, err)
+	require.NotEmpty(t, out.Addons)
+	for _, a := range out.Addons {
+		spec, ok := addon.Lookup(aws.StringValue(a.AddonName))
+		require.True(t, ok)
+		require.Len(t, a.AddonVersions, len(spec.Versions))
+		for i, v := range a.AddonVersions {
+			assert.Equal(t, "v"+spec.Versions[i]+"-eksbuild.1", aws.StringValue(v.AddonVersion),
+				"%s version %d", spec.Name, i)
+		}
+	}
+}
+
+func TestCreateAddon_AcceptsBothFormsAndStoresTheBareForm(t *testing.T) {
+	t.Parallel()
+	for name, input := range map[string]string{"aws form": csiAWSVersion, "bare form": csiBare} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			svc, fake := setupAddonService(t)
+
+			out, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+				AddonVersion: aws.String(input),
+			}, testAccountID)
+			require.NoError(t, err, notBeforeAWSOK)
+			assert.Equal(t, csiAWSVersion, aws.StringValue(out.Addon.AddonVersion), "response is in AWS form")
+
+			rec, err := addon.Get(t.Context(), acctKVForTest(t, svc), "c1", csiAddon)
+			require.NoError(t, err)
+			assert.Equal(t, csiBare, rec.AddonVersion, "the record keeps the upstream form")
+			require.Len(t, fake.installs, 1)
+			assert.Equal(t, csiBare, fake.installs[0].AddonVersion, "the bundle lookup sees the upstream form")
+		})
+	}
+}
+
+func TestCreateAddon_DefaultVersionResponseIsAWSForm(t *testing.T) {
+	t.Parallel()
+	svc, _ := setupAddonService(t)
+	out, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, csiAWSVersion, aws.StringValue(out.Addon.AddonVersion))
+}
+
+func TestUpdateAddon_AcceptsBothFormsAndStoresTheBareForm(t *testing.T) {
+	t.Parallel()
+	for name, input := range map[string]string{"aws form": csiAWSVersion, "bare form": csiBare} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			svc, fake := setupAddonService(t)
+			_, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+			}, testAccountID)
+			require.NoError(t, err)
+
+			_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+				AddonVersion: aws.String(input),
+			}, testAccountID)
+			require.NoError(t, err, notBeforeAWSOK)
+
+			rec, err := addon.Get(t.Context(), acctKVForTest(t, svc), "c1", csiAddon)
+			require.NoError(t, err)
+			assert.Equal(t, csiBare, rec.AddonVersion)
+			require.Len(t, fake.installs, 2)
+			assert.Equal(t, csiBare, fake.installs[1].AddonVersion)
+		})
+	}
+}
+
+func TestDescribeAddon_ReturnsAWSFormForAnyStoredForm(t *testing.T) {
+	t.Parallel()
+	for name, stored := range map[string]string{
+		"record stored bare before the change": csiBare,
+		"record already in AWS form":           csiAWSVersion,
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			svc, _ := setupAddonService(t)
+			_, err := svc.addons().Create(context.Background(), acctKVForTest(t, svc), testAccountID, addon.Desired{
+				Cluster: "c1", Name: csiAddon, Version: stored,
+			})
+			require.NoError(t, err)
+
+			desc, err := svc.DescribeAddon(context.Background(), &eks.DescribeAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+			}, testAccountID)
+			require.NoError(t, err)
+			assert.Equal(t, csiAWSVersion, aws.StringValue(desc.Addon.AddonVersion), "no double wrap")
+
+			list, err := svc.ListAddons(context.Background(), &eks.ListAddonsInput{ClusterName: aws.String("c1")}, testAccountID)
+			require.NoError(t, err)
+			assert.Equal(t, []string{csiAddon}, aws.StringValueSlice(list.Addons))
+		})
+	}
+}
+
+func TestUpdateAddon_ResponseAndRecordAfterBareRecord(t *testing.T) {
+	t.Parallel()
+	svc, _ := setupAddonService(t)
+	_, err := svc.addons().Create(context.Background(), acctKVForTest(t, svc), testAccountID, addon.Desired{
+		Cluster: "c1", Name: csiAddon, Version: csiBare,
+	})
+	require.NoError(t, err)
+
+	_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+		ConfigurationValues: aws.String(`{"a":1}`),
+	}, testAccountID)
+	require.NoError(t, err)
+	desc, err := svc.DescribeAddon(context.Background(), &eks.DescribeAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, csiAWSVersion, aws.StringValue(desc.Addon.AddonVersion))
+}
+
+func TestAddonVersionRefusal_NamesTheParameterAndListsSupportedValues(t *testing.T) {
+	t.Parallel()
+	svc, _ := setupAddonService(t)
+	_, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String("argocd"),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	refusals := map[string]func() error{
+		"create": func() error {
+			_, e := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+				AddonVersion: aws.String(csiWrongBuild),
+			}, testAccountID)
+			return e
+		},
+		"update": func() error {
+			_, e := svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String("argocd"),
+				AddonVersion: aws.String("v3.0.23-eksbuild.2"),
+			}, testAccountID)
+			return e
+		},
+	}
+	want := map[string]string{"create": csiAWSVersion, "update": "v3.0.23-eksbuild.1"}
+	for name, refuse := range refusals {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			err := refuse()
+			require.Error(t, err)
+			code, message, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok)
+			assert.Equal(t, awserrors.ErrorInvalidParameterValue, code, "this change leaves the error code alone")
+			assert.Contains(t, message, versionParam)
+			assert.Contains(t, message, want[name], "the supported AWS-form value is listed")
+		})
+	}
+}
+
+func TestAddonBareVersionInput_LogsItsUseAtInfo(t *testing.T) {
+	var buf bytes.Buffer
+	restore := slog.Default()
+	slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
+	t.Cleanup(func() { slog.SetDefault(restore) })
+
+	svc, _ := setupAddonService(t)
+	_, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	buf.Reset()
+	_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+		AddonVersion: aws.String(csiBare),
+	}, testAccountID)
+	require.NoError(t, err)
+	logged := buf.String()
+	assert.Contains(t, logged, "level=INFO")
+	assert.Contains(t, logged, "bare")
+	assert.Contains(t, logged, csiAddon)
+	assert.Contains(t, logged, "c1")
+
+	buf.Reset()
+	_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+		AddonVersion: aws.String(csiAWSVersion),
+	}, testAccountID)
+	require.NoError(t, err, notBeforeAWSOK)
+	assert.NotContains(t, buf.String(), "bare", "the AWS form is the intended form and logs nothing")
+}

spinifex/domains/eks/cluster/addons.go

@@ -2,8 +2,12 @@ package cluster
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
+	"log/slog"
+	"slices"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/eks"
@@ -35,15 +39,20 @@ func (s *EKSServiceImpl) ListAddons(ctx context.Context, input *eks.ListAddonsIn
 	return &eks.ListAddonsOutput{Addons: aws.StringSlice(names)}, nil
 }
 
-// DescribeAddonVersions returns the static add-on catalog, optionally filtered by name.
-func (s *EKSServiceImpl) DescribeAddonVersions(ctx context.Context, input *eks.DescribeAddonVersionsInput, _ string) (*eks.DescribeAddonVersionsOutput, error) {
-	filter := ""
-	if input != nil {
-		filter = aws.StringValue(input.AddonName)
+// DescribeAddonVersions returns the add-on catalogue filtered by name, owner,
+// publisher, type and the Kubernetes version its versions support, one page
+// of add-ons at a time.
+func (s *EKSServiceImpl) DescribeAddonVersions(_ context.Context, input *eks.DescribeAddonVersionsInput, _ string) (*eks.DescribeAddonVersionsOutput, error) {
+	if input == nil {
+		input = &eks.DescribeAddonVersionsInput{}
 	}
-	specs := addon.Specs()
-	out := make([]*eks.AddonInfo, 0, len(specs))
-	for _, spec := range specs {
+	// Every catalogue version supports only the K3s line the node image ships.
+	if v := aws.StringValue(input.KubernetesVersion); v != "" && v != defaultK8sVersion {
+		return &eks.DescribeAddonVersionsOutput{Addons: []*eks.AddonInfo{}}, nil
+	}
+	filter := aws.StringValue(input.AddonName)
+	var matched []addon.Spec
+	for _, spec := range addon.Specs() {
 		if filter != "" && spec.Name != filter {
 			continue
 		}
@@ -52,9 +61,51 @@ func (s *EKSServiceImpl) DescribeAddonVersions(ctx context.Context, input *eks.D
 		if spec.Hidden && filter == "" {
 			continue
 		}
+		if !matchesAny(input.Types, spec.Type) || !matchesAny(input.Owners, addonOwner) ||
+			!matchesAny(input.Publishers, addonPublisher) {
+			continue
+		}
+		matched = append(matched, spec)
+	}
+	page, next, err := pageAddons(matched, input.MaxResults, input.NextToken)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]*eks.AddonInfo, 0, len(page))
+	for _, spec := range page {
 		out = append(out, addonSpecToAWS(spec))
 	}
-	return &eks.DescribeAddonVersionsOutput{Addons: out}, nil
+	return &eks.DescribeAddonVersionsOutput{Addons: out, NextToken: next}, nil
+}
+
+// matchesAny reports whether the filter is empty or lists value.
+func matchesAny(filter []*string, value string) bool {
+	return len(filter) == 0 || slices.Contains(aws.StringValueSlice(filter), value)
+}
+
+// maxAddonPage bounds maxResults, the model's range being 1 to 100.
+const maxAddonPage = 100
+
+// pageAddons returns the add-ons after the token, at most maxResults of them.
+// The token is the base64 name of the last add-on returned, opaque to clients.
+func pageAddons(specs []addon.Spec, maxResults *int64, token *string) ([]addon.Spec, *string, error) {
+	if t := aws.StringValue(token); t != "" {
+		raw, err := base64.RawURLEncoding.DecodeString(t)
+		if err != nil {
+			return nil, nil, awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "nextToken is not valid")
+		}
+		after := string(raw)
+		specs = slices.DeleteFunc(slices.Clone(specs), func(sp addon.Spec) bool { return sp.Name <= after })
+	}
+	limit := len(specs)
+	if maxResults != nil {
+		limit = min(max(int(*maxResults), 1), maxAddonPage)
+	}
+	if limit >= len(specs) {
+		return specs, nil, nil
+	}
+	next := base64.RawURLEncoding.EncodeToString([]byte(specs[limit-1].Name))
+	return specs[:limit], &next, nil
 }
 
 // validateAddonServiceAccountRoleArn rejects a non-empty value that is not
@@ -89,11 +140,12 @@ func (s *EKSServiceImpl) CreateAddon(ctx context.Context, input *eks.CreateAddon
 	if !ok {
 		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 	}
-	version := aws.StringValue(input.AddonVersion)
-	if version == "" {
-		version = spec.DefaultVersion
-	} else if !spec.SupportsVersion(version) {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+	version := spec.DefaultVersion
+	if requested := aws.StringValue(input.AddonVersion); requested != "" {
+		var err error
+		if version, err = resolveAddonVersion(ctx, spec, requested, cluster); err != nil {
+			return nil, err
+		}
 	}
 	acctKV, err := s.acctKVForNewDependant(ctx, accountID, cluster)
 	if err != nil {
@@ -197,9 +249,13 @@ func (s *EKSServiceImpl) UpdateAddon(ctx context.Context, input *eks.UpdateAddon
 	version := aws.StringValue(input.AddonVersion)
 	if version != "" {
 		spec, ok := addon.Lookup(addonName)
-		if !ok || !spec.SupportsVersion(version) {
+		if !ok {
 			return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 		}
+		var err error
+		if version, err = resolveAddonVersion(ctx, spec, version, cluster); err != nil {
+			return nil, err
+		}
 	}
 	rec, err := s.addons().Update(ctx, acctKV, accountID, cluster, addonName, addon.Change{
 		Version:               version,
@@ -246,7 +302,7 @@ func addonRecordToAWS(cluster string, rec *addon.Record) *eks.Addon {
 	out := &eks.Addon{
 		AddonArn:     aws.String(rec.Arn),
 		AddonName:    aws.String(rec.AddonName),
-		AddonVersion: aws.String(rec.AddonVersion),
+		AddonVersion: aws.String(awsAddonVersion(rec)),
 		ClusterName:  aws.String(cluster),
 		Status:       aws.String(string(rec.Status)),
 		CreatedAt:    aws.Time(rec.CreatedAt),
@@ -269,17 +325,36 @@ func addonRecordToAWS(cluster string, rec *addon.Record) *eks.Addon {
 	return out
 }
 
-// addonSpecToAWS converts a catalog spec to the SDK AddonInfo shape.
+// The owner and publisher Spinifex reports for every catalogue add-on, as AWS
+// does for the add-ons it publishes.
+const (
+	addonOwner     = "aws"
+	addonPublisher = "eks"
+)
+
+// addonSpecToAWS converts a catalog spec to the SDK AddonInfo shape. Every
+// version supports the one cluster version the node image ships, and exactly
+// one of them is that cluster version's default.
 func addonSpecToAWS(spec addon.Spec) *eks.AddonInfo {
 	versions := make([]*eks.AddonVersionInfo, 0, len(spec.Versions))
-	for _, v := range spec.Versions {
+	for _, v := range spec.AWSVersions() {
 		versions = append(versions, &eks.AddonVersionInfo{
 			AddonVersion:           aws.String(v),
+			Architecture:           aws.StringSlice([]string{"amd64"}),
+			RequiresConfiguration:  aws.Bool(false),
 			RequiresIamPermissions: aws.Bool(spec.RequiresIRSA),
+			Compatibilities: []*eks.Compatibility{{
+				ClusterVersion:   aws.String(defaultK8sVersion),
+				PlatformVersions: aws.StringSlice([]string{"*"}),
+				DefaultVersion:   aws.Bool(v == spec.AWSVersion(spec.DefaultVersion)),
+			}},
 		})
 	}
 	return &eks.AddonInfo{
 		AddonName:     aws.String(spec.Name),
+		Type:          aws.String(spec.Type),
+		Owner:         aws.String(addonOwner),
+		Publisher:     aws.String(addonPublisher),
 		AddonVersions: versions,
 	}
 }
@@ -308,3 +383,30 @@ func (s *EKSServiceImpl) addonBucket(ctx context.Context, accountID string) (jet
 	}
 	return GetOrCreateAccountBucket(ctx, js, accountID)
 }
+
+// resolveAddonVersion maps a client version, in AWS form or bare upstream form,
+// to the upstream form the record and the bundle lookup use. The bare form is
+// more lenient than AWS, so each use is logged.
+func resolveAddonVersion(ctx context.Context, spec addon.Spec, requested, cluster string) (string, error) {
+	upstream, ok := spec.ParseVersion(requested)
+	if !ok {
+		return "", awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+			"addonVersion %q is not supported for add-on %s; supported versions: %s",
+			requested, spec.Name, strings.Join(spec.AWSVersions(), ", "))
+	}
+	if requested == upstream {
+		slog.InfoContext(ctx, "EKS add-on version given in bare upstream form",
+			"addon", spec.Name, "cluster", cluster, "version", requested)
+	}
+	return upstream, nil
+}
+
+// awsAddonVersion returns a stored record's version in AWS form; an add-on the
+// catalogue no longer lists keeps its stored string.
+func awsAddonVersion(rec *addon.Record) string {
+	spec, ok := addon.Lookup(rec.AddonName)
+	if !ok {
+		return rec.AddonVersion
+	}
+	return spec.AWSVersion(rec.AddonVersion)
+}

spinifex/domains/eks/cluster/addons_test.go

@@ -98,7 +98,9 @@ func TestCreateAddon_UnknownVersionRejected(t *testing.T) {
 		ClusterName: aws.String("c1"), AddonName: aws.String(albController),
 		AddonVersion: aws.String("9.9.9-nope"),
 	}, testAccountID)
-	require.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
 }
 
 func TestCreateAddon_CreatesStagingAndDefaultsVersion(t *testing.T) {
@@ -113,7 +115,7 @@ func TestCreateAddon_CreatesStagingAndDefaultsVersion(t *testing.T) {
 	require.NotNil(t, out.Addon)
 	// Version defaults to catalog default; status starts CREATING (honest until
 	// VM-side delivery confirms ACTIVE).
-	assert.Equal(t, spec.DefaultVersion, aws.StringValue(out.Addon.AddonVersion))
+	assert.Equal(t, spec.AWSVersion(spec.DefaultVersion), aws.StringValue(out.Addon.AddonVersion))
 	assert.Equal(t, string(addon.StatusCreating), aws.StringValue(out.Addon.Status))
 	assert.Contains(t, aws.StringValue(out.Addon.AddonArn), ":addon/c1/"+albController)
 
@@ -214,7 +216,7 @@ func TestUpdateAddon_ChangesVersionAndReinstalls(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 	assert.Equal(t, `{"replicaCount":2}`, aws.StringValue(desc.Addon.ConfigurationValues))
-	assert.Equal(t, spec.DefaultVersion, aws.StringValue(desc.Addon.AddonVersion))
+	assert.Equal(t, spec.AWSVersion(spec.DefaultVersion), aws.StringValue(desc.Addon.AddonVersion))
 	assert.Equal(t, string(addon.StatusUpdating), aws.StringValue(desc.Addon.Status))
 }
 
@@ -230,7 +232,9 @@ func TestUpdateAddon_UnknownVersionRejected(t *testing.T) {
 		ClusterName: aws.String("c1"), AddonName: aws.String("argocd"),
 		AddonVersion: aws.String("9.9.9-nope"),
 	}, testAccountID)
-	require.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
 }
 
 func TestUpdateAddon_MissingIsNotFound(t *testing.T) {

spinifex/domains/eks/cluster/cluster_security_group_test.go

@@ -0,0 +1,421 @@
+package cluster
+
+import (
+	"context"
+	"maps"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// sgCreatesIn returns the CreateSecurityGroup calls made against vpc whose name matches.
+func sgCreatesIn(f *fakeSGProvisioner, vpc string, match func(name string) bool) []*ec2.CreateSecurityGroupInput {
+	var out []*ec2.CreateSecurityGroupInput
+	for _, c := range f.createCalls {
+		if aws.StringValue(c.VpcId) == vpc && match(aws.StringValue(c.GroupName)) {
+			out = append(out, c)
+		}
+	}
+	return out
+}
+
+func sgCreateTags(c *ec2.CreateSecurityGroupInput) map[string]string {
+	m := map[string]string{}
+	for _, spec := range c.TagSpecifications {
+		maps.Copy(m, sgTagMap(spec.Tags))
+	}
+	return m
+}
+
+func authorizedOn(f *fakeSGProvisioner, sgID string) []*ec2.IpPermission {
+	var out []*ec2.IpPermission
+	for _, c := range f.authorizeCalls {
+		if aws.StringValue(c.GroupId) == sgID {
+			out = append(out, c.IpPermissions...)
+		}
+	}
+	return out
+}
+
+func deletedSGIDs(f *fakeSGProvisioner) []string {
+	var out []string
+	for _, c := range f.deleteCalls {
+		out = append(out, aws.StringValue(c.GroupId))
+	}
+	return out
+}
+
+// seedClusterSG registers an AWS-style cluster SG the way a created one looks.
+func seedClusterSG(f *fakeSGProvisioner, cluster, vpc, id string) {
+	f.existing[clusterSGNamePrefix+cluster+"-0123456789abcdef|"+vpc] = id
+	if f.tagsByID == nil {
+		f.tagsByID = map[string][]*ec2.Tag{}
+	}
+	f.tagsByID[id] = []*ec2.Tag{
+		{Key: aws.String("kubernetes.io/cluster/" + cluster), Value: aws.String("owned")},
+		{Key: aws.String("aws:eks:cluster-name"), Value: aws.String(cluster)},
+		{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(cluster)},
+		{Key: aws.String(clusterEKSRoleTagKey), Value: aws.String("cluster")},
+		{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)},
+	}
+}
+
+// seedCustomerSG registers a customer-made SG, optionally tagged to imitate ours.
+func seedCustomerSG(f *fakeSGProvisioner, name, vpc, id string, tg []*ec2.Tag) {
+	f.existing[name+"|"+vpc] = id
+	if f.tagsByID == nil {
+		f.tagsByID = map[string][]*ec2.Tag{}
+	}
+	f.tagsByID[id] = tg
+}
+
+// spoofTags are every tag a customer can set that the service also stamps.
+func spoofTags(cluster string) []*ec2.Tag {
+	return []*ec2.Tag{
+		{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(cluster)},
+		{Key: aws.String(clusterEKSRoleTagKey), Value: aws.String("cluster")},
+		{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)},
+		{Key: aws.String("kubernetes.io/cluster/" + cluster), Value: aws.String("owned")},
+	}
+}
+
+func setStoredClusterSG(t *testing.T, f *deleteClusterFixture, cluster, id string) {
+	t.Helper()
+	meta, err := GetClusterMeta(t.Context(), f.kv, cluster)
+	require.NoError(t, err)
+	meta.ResourcesVpcConfig.ClusterSecurityGroupId = id
+	require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta))
+}
+
+func createAlpha(t *testing.T, mutate func(*eks.CreateClusterInput)) (*eksServiceFixture, *ClusterMeta) {
+	t.Helper()
+	f := newEKSServiceFixture(t)
+	f.sg.uniqueIDs = true
+	in := createInput("alpha")
+	if mutate != nil {
+		mutate(in)
+	}
+	_, err := f.svc.CreateCluster(context.Background(), in, testAccountID, "")
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+	meta, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+	require.NoError(t, err)
+	require.NotNil(t, meta.ResourcesVpcConfig)
+	return f, meta
+}
+
+func withPrivateAccess(in *eks.CreateClusterInput) {
+	in.ResourcesVpcConfig.EndpointPrivateAccess = aws.Bool(true)
+}
+
+// CreateCluster makes the one AWS-style cluster SG in the caller's VPC and
+// reports it as clusterSecurityGroupId.
+func TestCreateCluster_ClusterSGHasAWSNameTagsAndRules(t *testing.T) {
+	t.Parallel()
+	f, meta := createAlpha(t, nil)
+
+	created := sgCreatesIn(f.sg, "vpc-aaa", func(n string) bool { return strings.HasPrefix(n, clusterSGNamePrefix+"alpha-") })
+	require.Len(t, created, 1, "exactly one eks-cluster-sg-<name>-<id> SG in the customer VPC")
+	name := aws.StringValue(created[0].GroupName)
+	assert.NotEmpty(t, strings.TrimPrefix(name, clusterSGNamePrefix+"alpha-"), "name carries a unique id suffix")
+
+	tg := sgCreateTags(created[0])
+	assert.Equal(t, "owned", tg["kubernetes.io/cluster/alpha"])
+	assert.Equal(t, "alpha", tg["aws:eks:cluster-name"])
+	assert.NotEmpty(t, tg["Name"])
+	assert.Equal(t, "alpha", tg[clusterEKSClusterTagKey])
+	assert.Equal(t, tags.ManagedByEKS, tg[tags.ManagedByKey])
+
+	id := f.sg.existing[name+"|vpc-aaa"]
+	require.NotEmpty(t, id)
+	assert.Equal(t, id, meta.ResourcesVpcConfig.ClusterSecurityGroupId)
+
+	perms := authorizedOn(f.sg, id)
+	require.Len(t, perms, 1, "inbound all from itself, nothing else")
+	assert.Equal(t, "-1", aws.StringValue(perms[0].IpProtocol))
+	require.Len(t, perms[0].UserIdGroupPairs, 1)
+	assert.Equal(t, id, aws.StringValue(perms[0].UserIdGroupPairs[0].GroupId))
+	for _, c := range f.sg.revokeEgressCall {
+		assert.NotEqual(t, id, aws.StringValue(c.GroupId), "egress stays allow-all")
+	}
+
+	out, err := f.svc.DescribeCluster(context.Background(), &eks.DescribeClusterInput{Name: aws.String("alpha")}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, id, aws.StringValue(out.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId))
+}
+
+// A new cluster has no customer-VPC control-plane or nodegroup SG, and exactly
+// one SG per cluster carries kubernetes.io/cluster/<name>.
+func TestCreateCluster_NoSeparateCustomerVPCControlPlaneOrNodegroupSG(t *testing.T) {
+	t.Parallel()
+	f, _ := createAlpha(t, withPrivateAccess)
+
+	retired := sgCreatesIn(f.sg, "vpc-aaa", func(n string) bool {
+		return n == ClusterControlPlaneSGName("alpha") || n == ClusterNodegroupSGName("alpha")
+	})
+	assert.Empty(t, retired, "customer VPC must not get control-plane-sg or nodegroup-sg")
+
+	tagged := 0
+	for _, c := range f.sg.createCalls {
+		if _, ok := sgCreateTags(c)["kubernetes.io/cluster/alpha"]; ok {
+			tagged++
+		}
+	}
+	assert.Equal(t, 1, tagged, "exactly one SG tagged kubernetes.io/cluster/alpha")
+}
+
+// The private-endpoint ENI carries its own SG and the cluster SG.
+func TestCreateCluster_PrivateEndpointENICarriesBothSGs(t *testing.T) {
+	t.Parallel()
+	f, meta := createAlpha(t, withPrivateAccess)
+
+	var eni *ec2.CreateNetworkInterfaceInput
+	for _, c := range f.vpc.createCalls {
+		if strings.Contains(aws.StringValue(c.Description), "private-endpoint") {
+			eni = c
+		}
+	}
+	require.NotNil(t, eni, "private-endpoint ENI created")
+	peSG := f.sg.existing[ClusterPrivateEndpointSGName("alpha")+"|vpc-aaa"]
+	require.NotEmpty(t, peSG)
+	assert.ElementsMatch(t,
+		[]string{peSG, meta.ResourcesVpcConfig.ClusterSecurityGroupId},
+		aws.StringValueSlice(eni.Groups))
+	assert.NotContains(t, sgTagMap(f.sg.tagsByID[peSG]), "kubernetes.io/cluster/alpha",
+		"only the cluster SG carries the cluster tag")
+}
+
+// The system control-plane VPC keeps its control-plane SG and rules.
+func TestCreateCluster_SystemCPVPCSecurityGroupUnchanged(t *testing.T) {
+	t.Parallel()
+	f, meta := createAlpha(t, nil)
+	require.NotNil(t, meta.ManagedCPVPC)
+	cpVPC := meta.ManagedCPVPC.VpcId
+
+	cp := sgCreatesIn(f.sg, cpVPC, func(n string) bool { return n == ClusterControlPlaneSGName("alpha") })
+	require.Len(t, cp, 1)
+	cpID := f.sg.existing[ClusterControlPlaneSGName("alpha")+"|"+cpVPC]
+	require.NotEmpty(t, cpID)
+
+	var ports []int64
+	selfRef := false
+	for _, p := range authorizedOn(f.sg, cpID) {
+		ports = append(ports, aws.Int64Value(p.FromPort))
+		for _, pair := range p.UserIdGroupPairs {
+			if aws.StringValue(pair.GroupId) == cpID {
+				selfRef = true
+			}
+		}
+	}
+	assert.Subset(t, ports, []int64{k3sAPIServerPort, konnectivityAgentPort, webhookReadyPort, 2379, 10250})
+	assert.True(t, selfRef, "HA self-referencing rules stay")
+	assert.Empty(t, sgCreatesIn(f.sg, cpVPC, func(n string) bool { return strings.HasPrefix(n, clusterSGNamePrefix) }),
+		"the cluster SG is a customer-VPC resource only")
+}
+
+// Workers launch with exactly the cluster SG and no extra SG or cross rules.
+func TestCreateNodegroup_WorkersGetExactlyTheClusterSG(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	f.sg.uniqueIDs = true
+	seedActiveClusterWithToken(t, f, "c1")
+	meta, err := GetClusterMeta(t.Context(), f.kv, "c1")
+	require.NoError(t, err)
+	meta.ResourcesVpcConfig.ClusterSecurityGroupId = "sg-cluster"
+	require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta))
+	seedClusterSG(f.sg, "c1", "vpc-aaa", "sg-cluster")
+	markWorkersReady(t, f, "c1", "ng1", 2)
+
+	_, err = f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 2), testAccountID)
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+
+	require.Len(t, f.worker.runCalls, 2)
+	for _, c := range f.worker.runCalls {
+		assert.Equal(t, []string{"sg-cluster"}, aws.StringValueSlice(c.SecurityGroupIds))
+	}
+	assert.Empty(t, f.sg.createCalls, "no nodegroup or control-plane SG is created for a worker")
+	assert.Empty(t, f.sg.authorizeCalls, "no CP<->nodegroup cross rules")
+}
+
+// A launch template that names security groups gets exactly those.
+func TestLaunchOneWorker_TemplateSecurityGroupsReplaceClusterSG(t *testing.T) {
+	t.Parallel()
+	reader := &fakeLaunchTemplates{out: &ec2.DescribeLaunchTemplateVersionsOutput{
+		LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{{
+			LaunchTemplateData: &ec2.ResponseLaunchTemplateData{
+				SecurityGroupIds: aws.StringSlice([]string{"sg-lt-1", "sg-lt-2"}),
+			},
+		}},
+	}}
+	f, err := launchWorkerWith(t, &NodegroupLaunchTemplate{ID: "lt-0123", Version: "1"}, reader)
+	require.NoError(t, err)
+
+	require.Len(t, f.worker.runCalls, 1)
+	assert.Equal(t, []string{"sg-lt-1", "sg-lt-2"}, aws.StringValueSlice(f.worker.runCalls[0].SecurityGroupIds))
+}
+
+// A template naming no security groups falls back to the cluster SG alone.
+func TestLaunchOneWorker_TemplateWithoutSecurityGroupsUsesClusterSG(t *testing.T) {
+	t.Parallel()
+	f, err := launchWorkerWith(t, &NodegroupLaunchTemplate{ID: "lt-0123", Version: "1"},
+		&fakeLaunchTemplates{out: templateVersionWith(nil)})
+	require.NoError(t, err)
+
+	require.Len(t, f.worker.runCalls, 1)
+	assert.Equal(t, []string{"sg-ng"}, aws.StringValueSlice(f.worker.runCalls[0].SecurityGroupIds))
+}
+
+// Delete removes the SG recorded on the cluster, whatever its name, and leaves
+// another cluster's SG in the same VPC alone.
+func TestDeleteCluster_RemovesClusterSGFoundByTag(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	seedClusterSG(f.sg, "alpha", "vpc-aaa", "sg-new")
+	seedClusterSG(f.sg, "beta", "vpc-aaa", "sg-beta")
+	setStoredClusterSG(t, f, "alpha", "sg-new")
+	f.sg.perms = map[string][]*ec2.IpPermission{"sg-new": {{
+		IpProtocol:       aws.String("-1"),
+		UserIdGroupPairs: []*ec2.UserIdGroupPair{{GroupId: aws.String("sg-new")}},
+	}}}
+	f.sg.enforceDepViolation = true
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	deleted := deletedSGIDs(f.sg)
+	assert.Contains(t, deleted, "sg-new")
+	assert.NotContains(t, deleted, "sg-beta")
+	_, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+	assert.Error(t, err, "teardown completed and removed the record")
+}
+
+// An old-layout cluster (control-plane-sg + nodegroup-sg, no cluster SG) still
+// has both SGs removed.
+func TestDeleteCluster_OldLayoutStillRemovesBothNamedSGs(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	f.sg.existing[ClusterControlPlaneSGName("alpha")+"|vpc-aaa"] = "sg-old-cp"
+	f.sg.existing[ClusterNodegroupSGName("alpha")+"|vpc-aaa"] = "sg-old-ng"
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	deleted := deletedSGIDs(f.sg)
+	assert.Contains(t, deleted, "sg-old-cp")
+	assert.Contains(t, deleted, "sg-old-ng")
+}
+
+// A cluster SG that cannot be deleted yet keeps the cluster DELETING; the
+// deleting reaper removes it once the dependency clears, not the request.
+func TestDeleteCluster_ClusterSGConvergesInTheReaper(t *testing.T) {
+	origBudget, origInterval := sgDeleteWaitBudget, sgDeleteWaitInterval
+	sgDeleteWaitBudget, sgDeleteWaitInterval = 10*time.Millisecond, time.Millisecond
+	t.Cleanup(func() { sgDeleteWaitBudget, sgDeleteWaitInterval = origBudget, origInterval })
+
+	f := newDeleteClusterFixture(t, "alpha")
+	seedClusterSG(f.sg, "alpha", "vpc-aaa", "sg-new")
+	setStoredClusterSG(t, f, "alpha", "sg-new")
+	f.sg.deleteFailsBefore = map[string]int{"sg-new": 1000}
+
+	deleteAndSettle(t, f.svc, "alpha")
+	meta := requireTeardownRetained(t, f.kv, "alpha")
+	assert.Contains(t, meta.DeleteFailedStep, stepSecurityGroups)
+
+	f.sg.mu.Lock()
+	f.sg.deleteFailsBefore["sg-new"] = 0
+	f.sg.mu.Unlock()
+	redrive(t, f.svc)
+
+	_, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+	assert.Error(t, err, "the reaper finished the teardown")
+}
+
+// A customer-made SG carrying the tags the service stamps is neither adopted by
+// a create retry nor deleted with the cluster.
+func TestEnsureClusterSG_DoesNotAdoptACustomerTaggedSG(t *testing.T) {
+	t.Parallel()
+	sg := newFakeSGProvisioner()
+	sg.uniqueIDs = true
+	seedCustomerSG(sg, "customer-sg", "vpc-aaa", "sg-spoof", spoofTags("c1"))
+
+	id, err := EnsureClusterSG(context.Background(), sg, testAccountID, "c1", "vpc-aaa")
+	require.NoError(t, err)
+
+	assert.NotEqual(t, "sg-spoof", id)
+	assert.Len(t, sg.createCalls, 1)
+	for _, p := range sg.authorizeCalls {
+		assert.NotEqual(t, "sg-spoof", aws.StringValue(p.GroupId))
+	}
+}
+
+func TestDeleteCluster_DoesNotDeleteACustomerTaggedSG(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	seedClusterSG(f.sg, "alpha", "vpc-aaa", "sg-new")
+	setStoredClusterSG(t, f, "alpha", "sg-new")
+	seedCustomerSG(f.sg, "customer-sg", "vpc-aaa", "sg-spoof", spoofTags("alpha"))
+	f.sg.perms = map[string][]*ec2.IpPermission{"sg-spoof": {{
+		IpProtocol: aws.String("tcp"), FromPort: aws.Int64(22), ToPort: aws.Int64(22),
+		IpRanges: []*ec2.IpRange{{CidrIp: aws.String("10.0.0.0/8")}},
+	}}}
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	assert.Contains(t, deletedSGIDs(f.sg), "sg-new")
+	assert.NotContains(t, deletedSGIDs(f.sg), "sg-spoof")
+	for _, r := range f.sg.revokeCalls {
+		assert.NotEqual(t, "sg-spoof", aws.StringValue(r.GroupId), "no rule is revoked on it")
+	}
+}
+
+// Deleting a cluster revokes, on an unrelated SG in the VPC, only the rules that
+// reference a cluster-owned SG.
+func TestDeleteClusterSGs_UnrelatedSGKeepsItsOtherRules(t *testing.T) {
+	t.Parallel()
+	sgp := newFakeSGProvisioner()
+	seedClusterSG(sgp, "alpha", "vpc-aaa", "sg-new")
+	seedCustomerSG(sgp, "customer-sg", "vpc-aaa", "sg-other", nil)
+	ssh := &ec2.IpPermission{
+		IpProtocol: aws.String("tcp"), FromPort: aws.Int64(22), ToPort: aws.Int64(22),
+		IpRanges: []*ec2.IpRange{{CidrIp: aws.String("10.0.0.0/8")}},
+	}
+	mixed := &ec2.IpPermission{
+		IpProtocol: aws.String("tcp"), FromPort: aws.Int64(443), ToPort: aws.Int64(443),
+		IpRanges: []*ec2.IpRange{{CidrIp: aws.String("192.168.0.0/16")}},
+		UserIdGroupPairs: []*ec2.UserIdGroupPair{
+			{GroupId: aws.String("sg-new")},
+			{GroupId: aws.String("sg-peer")},
+		},
+	}
+	sgp.perms = map[string][]*ec2.IpPermission{"sg-other": {ssh, mixed}}
+	sgp.permsEgress = map[string][]*ec2.IpPermission{"sg-other": {groupRefPerm("tcp", 6443, "sg-new"), groupRefPerm("tcp", 80, "sg-peer")}}
+
+	require.NoError(t, DeleteClusterSGs(context.Background(), sgp, testAccountID, "alpha", "vpc-aaa", "sg-new"))
+
+	var revoked []*ec2.IpPermission
+	for _, r := range sgp.revokeCalls {
+		if aws.StringValue(r.GroupId) == "sg-other" {
+			revoked = append(revoked, r.IpPermissions...)
+		}
+	}
+	require.Len(t, revoked, 1, "only the rule referencing the cluster SG goes")
+	require.Len(t, revoked[0].UserIdGroupPairs, 1)
+	assert.Equal(t, "sg-new", aws.StringValue(revoked[0].UserIdGroupPairs[0].GroupId))
+	assert.Empty(t, revoked[0].IpRanges, "its CIDR sources are not part of the revoke")
+
+	var revokedEgress []*ec2.IpPermission
+	for _, r := range sgp.revokeEgressCall {
+		if aws.StringValue(r.GroupId) == "sg-other" {
+			revokedEgress = append(revokedEgress, r.IpPermissions...)
+		}
+	}
+	require.Len(t, revokedEgress, 1)
+	assert.Equal(t, "sg-new", aws.StringValue(revokedEgress[0].UserIdGroupPairs[0].GroupId))
+	assert.NotContains(t, deletedSGIDs(sgp), "sg-other")
+}

spinifex/domains/eks/cluster/create_cluster_test.go

@@ -419,37 +419,6 @@ func TestCreateCluster_BuildsManagedCPVPCUnderSystemAccount(t *testing.T) {
 	assert.True(t, tagged, "NAT gateway EIP must carry the ManagedBy=eks tag")
 }
 
-// The primary cluster SG (clusterSecurityGroupId) must be created in the
-// caller's own VPC, not the system-managed CP VPC the control-plane VM
-// actually lives in — otherwise the tenant's own DescribeSecurityGroups can
-// never resolve it. fakeSubnetResolver resolves every subnet to "vpc-aaa".
-func TestCreateCluster_ClusterSecurityGroupCreatedInCallerVPC(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-
-	_, err := f.svc.CreateCluster(context.Background(), createInput("alpha"), testAccountID, "")
-	require.NoError(t, err)
-	f.svc.WaitLaunches()
-
-	meta, err := GetClusterMeta(t.Context(), f.kv, "alpha")
-	require.NoError(t, err)
-	require.NotNil(t, meta.ResourcesVpcConfig)
-	assert.NotEmpty(t, meta.ResourcesVpcConfig.ClusterSecurityGroupId)
-
-	found := false
-	for _, call := range f.sg.createCalls {
-		if aws.StringValue(call.GroupName) == ClusterControlPlaneSGName("alpha") && aws.StringValue(call.VpcId) == "vpc-aaa" {
-			found = true
-		}
-	}
-	assert.True(t, found, "control-plane SG must be created in the caller's VPC (vpc-aaa)")
-
-	out, err := f.svc.DescribeCluster(context.Background(), &eks.DescribeClusterInput{Name: aws.String("alpha")}, testAccountID)
-	require.NoError(t, err)
-	require.NotNil(t, out.Cluster.ResourcesVpcConfig)
-	assert.Equal(t, meta.ResourcesVpcConfig.ClusterSecurityGroupId, aws.StringValue(out.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId))
-}
-
 // Security groups the caller explicitly passes on CreateCluster must come back
 // unchanged from DescribeCluster — never substituted with platform-created ids.
 func TestCreateCluster_CallerSecurityGroupIdsEchoedNotOverwritten(t *testing.T) {

spinifex/domains/eks/cluster/nodegroup.go

@@ -472,13 +472,9 @@ type nodegroupLaunchCtx struct {
 func (s *EKSServiceImpl) launchNodegroupInfra(ctx context.Context, lc nodegroupLaunchCtx) {
 	acctKV, accountID, cluster, ng, meta, rec := lc.acctKV, lc.accountID, lc.cluster, lc.ng, lc.meta, lc.rec
 
-	cpSGID, ngSGID, err := EnsureClusterSGs(ctx, s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId)
+	ngSGID, err := EnsureWorkerSG(ctx, s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId, meta.ResourcesVpcConfig.ClusterSecurityGroupId)
 	if err != nil {
-		s.markNodegroupFailed(ctx, acctKV, cluster, ng, "ensure cluster SGs: "+err.Error())
-		return
-	}
-	if err := EnsureNodegroupSGRules(ctx, s.deps.VPCSG, accountID, cluster, cpSGID, ngSGID); err != nil {
-		s.markNodegroupFailed(ctx, acctKV, cluster, ng, "ensure nodegroup SG rules: "+err.Error())
+		s.markNodegroupFailed(ctx, acctKV, cluster, ng, "ensure cluster SG: "+err.Error())
 		return
 	}
 
@@ -707,11 +703,15 @@ func (s *EKSServiceImpl) launchOneWorker(ctx context.Context, rec *NodegroupReco
 		}}
 	}
 
-	metadataOptions, err := s.workerMetadataOptions(ctx, rec, accountID)
+	metadataOptions, templateSGs, err := s.workerTemplateSettings(ctx, rec, accountID)
 	if err != nil {
 		return "", err
 	}
 	runInput.MetadataOptions = metadataOptions
+	// As on AWS, security groups named by the launch template replace the cluster SG.
+	if len(templateSGs) > 0 {
+		runInput.SecurityGroupIds = aws.StringSlice(templateSGs)
+	}
 
 	// Attach the node role's instance profile so IMDS serves the role to the ECR
 	// credential provider; without it worker pulls from the internal ECR get 401.
@@ -743,21 +743,23 @@ func (s *EKSServiceImpl) launchOneWorker(ctx context.Context, rec *NodegroupReco
 	return "", errors.New("run worker returned no instance id")
 }
 
-// workerMetadataOptions chooses a worker's IMDS options: fields a launch template
-// version sets win as given, and every unset field gets the managed nodegroup
-// default (IMDSv2 required, hop limit 2) so pods behind the node can reach IMDS.
-func (s *EKSServiceImpl) workerMetadataOptions(ctx context.Context, rec *NodegroupRecord, accountID string) (*ec2.InstanceMetadataOptionsRequest, error) {
+// workerTemplateSettings chooses a worker's IMDS options and security groups.
+// Metadata fields a launch template version sets win as given, and every unset
+// field gets the managed nodegroup default (IMDSv2 required, hop limit 2) so pods
+// behind the node can reach IMDS. The template's security group IDs, if any, are
+// returned for the caller to use in place of the cluster SG.
+func (s *EKSServiceImpl) workerTemplateSettings(ctx context.Context, rec *NodegroupRecord, accountID string) (*ec2.InstanceMetadataOptionsRequest, []string, error) {
 	managedDefault := &ec2.InstanceMetadataOptionsRequest{
 		HttpTokens:              aws.String(ec2.HttpTokensStateRequired),
 		HttpPutResponseHopLimit: aws.Int64(workerMetadataHopLimit),
 	}
 	if rec.LaunchTemplate == nil {
-		return managedDefault, nil
+		return managedDefault, nil, nil
 	}
 	if s.deps.LaunchTemplates == nil {
 		slog.ErrorContext(ctx, "EKS nodegroup: launch template named but no launch template service wired",
 			"nodegroup", rec.Name, "launchTemplate", rec.LaunchTemplate.ID+rec.LaunchTemplate.Name)
-		return nil, errors.New(awserrors.ErrorServerInternal)
+		return nil, nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
 	version := rec.LaunchTemplate.Version
@@ -772,14 +774,16 @@ func (s *EKSServiceImpl) workerMetadataOptions(ctx context.Context, rec *Nodegro
 	}
 	out, err := s.deps.LaunchTemplates.DescribeLaunchTemplateVersions(ctx, in, accountID)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	if len(out.LaunchTemplateVersions) == 0 || out.LaunchTemplateVersions[0].LaunchTemplateData == nil {
-		return nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound)
+		return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound)
 	}
-	opts := out.LaunchTemplateVersions[0].LaunchTemplateData.MetadataOptions
+	data := out.LaunchTemplateVersions[0].LaunchTemplateData
+	sgIDs := aws.StringValueSlice(data.SecurityGroupIds)
+	opts := data.MetadataOptions
 	if opts == nil {
-		return managedDefault, nil
+		return managedDefault, sgIDs, nil
 	}
 	req := &ec2.InstanceMetadataOptionsRequest{
 		HttpTokens:              opts.HttpTokens,
@@ -796,7 +800,7 @@ func (s *EKSServiceImpl) workerMetadataOptions(ctx context.Context, rec *Nodegro
 	if req.HttpPutResponseHopLimit == nil {
 		req.HttpPutResponseHopLimit = managedDefault.HttpPutResponseHopLimit
 	}
-	return req, nil
+	return req, sgIDs, nil
 }
 
 // selectWorkerHost picks the host for the next worker: the eligible host (one
@@ -886,7 +890,7 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 		return s.attachRoleToProfile(accountID, profileName, roleName,
 			aws.StringValue(out.InstanceProfile.Arn), len(out.InstanceProfile.Roles) > 0)
 	}
-	if err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
 		return "", fmt.Errorf("get instance profile %q: %w", profileName, err)
 	}
 
@@ -895,7 +899,7 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 	})
 	if err != nil {
 		// A racing worker launch may have created it first; re-read and converge.
-		if err.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMEntityAlreadyExists {
 			return "", fmt.Errorf("create instance profile %q: %w", profileName, err)
 		}
 		got, gerr := s.iamEnsurer().GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{
@@ -922,7 +926,7 @@ func (s *EKSServiceImpl) attachRoleToProfile(accountID, profileName, roleName, p
 		InstanceProfileName: aws.String(profileName),
 		RoleName:            aws.String(roleName),
 	})
-	if err != nil && err.Error() != awserrors.ErrorIAMLimitExceeded {
+	if code, _ := awserrors.ResolveErrorCode(err); err != nil && code != awserrors.ErrorIAMLimitExceeded {
 		return "", fmt.Errorf("add role %q to instance profile %q: %w", roleName, profileName, err)
 	}
 	return profileARN, nil
@@ -1092,9 +1096,9 @@ func (s *EKSServiceImpl) launchWorkersCAS(ctx context.Context, acctKV jetstream.
 		}
 		return nil, fmt.Errorf("resolve eks-node AMI: %w", err)
 	}
-	_, ngSGID, err := EnsureClusterSGs(ctx, s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId)
+	ngSGID, err := EnsureWorkerSG(ctx, s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId, meta.ResourcesVpcConfig.ClusterSecurityGroupId)
 	if err != nil {
-		return nil, fmt.Errorf("ensure cluster SGs: %w", err)
+		return nil, fmt.Errorf("ensure cluster SG: %w", err)
 	}
 
 	// Each iteration records at most one worker, so bound by the gap plus CAS slack.

spinifex/domains/eks/cluster/nodegroup_test.go

@@ -338,6 +338,25 @@ func TestEnsureNodegroupSGRules_DuplicateRuleTolerated(t *testing.T) {
 		"duplicate-rule error must be treated as success")
 }
 
+// A cluster created before the cluster SG keeps launching workers into its
+// nodegroup SG with the control-plane cross rules.
+func TestCreateNodegroup_OldLayoutClusterKeepsNodegroupSG(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	f.sg.createIDs = []string{"sg-old-cp", "sg-old-ng"}
+	seedActiveClusterWithToken(t, f, "c1")
+	f.sg.existing[ClusterNodegroupSGName("c1")+"|vpc-aaa"] = "sg-old-ng"
+	markWorkersReady(t, f, "c1", "ng1", 1)
+
+	_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+
+	require.Len(t, f.worker.runCalls, 1)
+	assert.Equal(t, []string{"sg-old-ng"}, aws.StringValueSlice(f.worker.runCalls[0].SecurityGroupIds))
+	assert.Len(t, f.sg.authorizeCalls, 6, "the cp<->ng cross rules are still ensured")
+}
+
 func TestCreateNodegroup_HappyPath(t *testing.T) {
 	t.Parallel()
 	f := newEKSServiceFixture(t)
@@ -382,8 +401,14 @@ func TestCreateNodegroup_HappyPath(t *testing.T) {
 		}
 	}
 
-	// SG ingress rules were authorized.
+	// A cluster with no SG yet gets the cluster SG, created once with its self
+	// rule, and every worker carries it.
+	require.Len(t, f.sg.createCalls, 1)
+	clusterSG := f.sg.existing[aws.StringValue(f.sg.createCalls[0].GroupName)+"|vpc-aaa"]
 	assert.NotEmpty(t, f.sg.authorizeCalls)
+	for _, call := range f.worker.runCalls {
+		assert.Equal(t, []string{clusterSG}, aws.StringValueSlice(call.SecurityGroupIds))
+	}
 
 	// Duplicate create → ResourceInUseException.
 	_, err = f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 2), testAccountID)

spinifex/domains/eks/cluster/private_endpoint.go

@@ -30,7 +30,7 @@ func EnsurePrivateEndpointENI(ctx context.Context,
 	vpcSvc k3sVPCProvisioner,
 	sgp sgProvisioner,
 	resolver SubnetVPCResolver,
-	accountID, clusterName, subnetID, vpcID string,
+	accountID, clusterName, subnetID, vpcID, clusterSGID string,
 ) (*PrivateEndpointENI, error) {
 	if accountID == "" {
 		return nil, errors.New("eks: EnsurePrivateEndpointENI empty account id")
@@ -41,6 +41,9 @@ func EnsurePrivateEndpointENI(ctx context.Context,
 	if vpcID == "" {
 		return nil, errors.New("eks: EnsurePrivateEndpointENI empty vpc id")
 	}
+	if clusterSGID == "" {
+		return nil, errors.New("eks: EnsurePrivateEndpointENI empty cluster SG id")
+	}
 
 	vpcCIDR, err := resolver.GetVPCCIDR(ctx, accountID, vpcID)
 	if err != nil {
@@ -55,7 +58,7 @@ func EnsurePrivateEndpointENI(ctx context.Context,
 	eniOut, err := vpcSvc.CreateNetworkInterface(ctx, &ec2.CreateNetworkInterfaceInput{
 		SubnetId:    aws.String(subnetID),
 		Description: aws.String("EKS private-endpoint ENI for " + clusterName),
-		Groups:      aws.StringSlice([]string{sgID}),
+		Groups:      aws.StringSlice([]string{sgID, clusterSGID}),
 		TagSpecifications: []*ec2.TagSpecification{{
 			ResourceType: aws.String("network-interface"),
 			Tags: []*ec2.Tag{

spinifex/domains/eks/cluster/private_endpoint_test.go

@@ -132,30 +132,32 @@ func TestEnsurePrivateEndpointENI_HappyPath(t *testing.T) {
 	sgp := newFakeSGProvisioner()
 	sgp.createIDs = []string{"sg-pe-001"}
 
-	pe, err := EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "vpc-aaa")
+	pe, err := EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "vpc-aaa", "sg-cluster")
 	require.NoError(t, err)
 	assert.Equal(t, "eni-pe-001", pe.ENIID)
 	assert.Equal(t, "10.0.1.50", pe.ENIIP)
 	assert.Equal(t, "02:0a:01:23:45:67", pe.ENIMac)
 	assert.Equal(t, "sg-pe-001", pe.SGID)
 
-	// ENI must be created in the customer subnet, bound to the private-endpoint SG.
+	// ENI must be created in the customer subnet, bound to the private-endpoint SG
+	// and the cluster SG.
 	require.Len(t, vpcSvc.createCalls, 1)
 	createIn := vpcSvc.createCalls[0]
 	assert.Equal(t, "subnet-aaa", aws.StringValue(createIn.SubnetId))
-	require.Len(t, createIn.Groups, 1)
-	assert.Equal(t, "sg-pe-001", aws.StringValue(createIn.Groups[0]))
+	assert.Equal(t, []string{"sg-pe-001", "sg-cluster"}, aws.StringValueSlice(createIn.Groups))
 }
 
 func TestEnsurePrivateEndpointENI_EmptyInputsRejected(t *testing.T) {
 	t.Parallel()
 	vpcSvc := &fakeK3sVPC{}
 	sgp := newFakeSGProvisioner()
-	_, err := EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "", "alpha", "subnet-aaa", "vpc-aaa")
+	_, err := EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "", "alpha", "subnet-aaa", "vpc-aaa", "sg-cluster")
 	require.Error(t, err)
-	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "", "vpc-aaa")
+	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "", "vpc-aaa", "sg-cluster")
 	require.Error(t, err)
-	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "")
+	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "", "sg-cluster")
+	require.Error(t, err)
+	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "vpc-aaa", "")
 	require.Error(t, err)
 	assert.Empty(t, vpcSvc.createCalls)
 }