feat(operator): record ADR-0008:S9 quiescence evidence on the deletion job (#1447)

* feat(iam): record quiescence evidence on the deletion job, with the L09 scope unresolved until the final inventory re-read

* fix(iam): cite ADR-0008:S9 as the basis for the STS consumer

.gitignore

@@ -24,6 +24,9 @@ build/microvm/qboot.LICENSE
 build/microvm/sqs-worker/vmlinuz
 build/microvm/sqs-worker/initramfs.cpio.gz
 build/microvm/sqs-worker/.stage.*/
+build/microvm/lambda-runtime/vmlinuz
+build/microvm/lambda-runtime/initramfs.cpio.gz
+build/microvm/lambda-runtime/.stage.*/
 build/qboot/
 !spinifex/runtime/roles/spinifexui/frontend/dist/
 tmp/

Makefile

@@ -169,7 +169,7 @@ go_local_srcs = $(shell GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go list
 # worker as separate processes. A separate artifact from the LB image above.
 SQS_WORKER_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/sqs-worker
 SQS_WORKER_MICROVM_ARTIFACTS := $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz $(SQS_WORKER_MICROVM_OUT_DIR)/initramfs.cpio.gz
-SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_WORKER_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-store sqs-worker-ready sqs-worker-stop) bin/nats-server bin/sqs-worker
+SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_WORKER_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-store sqs-worker-ready sqs-worker-stop) bin/nats-server bin/sqs-worker
 
 $(SQS_WORKER_MICROVM_ARTIFACTS) &: $(SQS_WORKER_MICROVM_INPUTS)
 	./scripts/build-sqs-store-microvm-image.sh
@@ -180,6 +180,20 @@ bin/sqs-worker: go.mod go.sum $$(call go_local_srcs,./cmd/sqs-worker)
 
 build-sqs-worker-microvm-image: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Build the SQS microVM kernel and initramfs
 
+# The Lambda runtime microVM image. Built on demand; not shipped or installed.
+LAMBDA_RUNTIME_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/lambda-runtime
+LAMBDA_RUNTIME_MICROVM_ARTIFACTS := $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/vmlinuz $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/initramfs.cpio.gz
+LAMBDA_RUNTIME_MICROVM_INPUTS := scripts/build-lambda-runtime-microvm-image.sh scripts/lib/microvm-image.sh $(addprefix $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/,init.sh inittab run-agent) bin/lambda-agent
+
+$(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) &: $(LAMBDA_RUNTIME_MICROVM_INPUTS)
+	./scripts/build-lambda-runtime-microvm-image.sh
+
+bin/lambda-agent: go.mod go.sum $$(call go_local_srcs,./cmd/lambda-agent)
+	@mkdir -p bin
+	GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/lambda-agent
+
+build-lambda-runtime-microvm-image: $(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) ## Build the Lambda runtime microVM kernel and initramfs (not shipped)
+
 # qboot is the microvm firmware. The source tarball ships with each release
 # because qboot is GPL-2.0.
 QBOOT_COMMIT := $(shell sed -n 's/^QBOOT_COMMIT="\(.*\)"$$/\1/p' scripts/build-qboot.sh)
@@ -522,7 +536,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
+.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

build/microvm/lambda-runtime/init.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+mount -t proc  proc  /proc
+mount -t sysfs sys   /sys
+mount -t devtmpfs dev /dev
+
+# modprobe needs a usable modules.dep; insmod by explicit path does not, so
+# fall back to it. Handles .ko and .ko.gz.
+load_mod() {
+    modprobe "$1" 2>/dev/null && return 0
+    for ko in $(find /lib/modules -name "$1.ko" -o -name "$1.ko.gz" 2>/dev/null); do
+        insmod "$ko" 2>/dev/null && return 0
+        case "$ko" in
+            *.gz) gunzip -c "$ko" >/tmp/mod.ko 2>/dev/null \
+                  && insmod /tmp/mod.ko 2>/dev/null && return 0 ;;
+        esac
+    done
+    return 1
+}
+load_mod virtio_net || true
+
+ip link set lo up
+
+exec /sbin/init

build/microvm/lambda-runtime/inittab

@@ -0,0 +1,6 @@
+# The power button loads here, not in /init: busybox init has its SIGUSR2
+# (poweroff) handler installed before sysinit runs, so a press can never reach
+# the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
+::sysinit:/sbin/modprobe tiny_power_button power_signal=12
+::once:/usr/local/lib/lambda-runtime/run-agent
+::ctrlaltdel:/sbin/reboot

build/microvm/lambda-runtime/run-agent

@@ -0,0 +1,6 @@
+#!/bin/sh
+# Runs the Lambda guest agent, then powers the guest off whatever its status.
+/usr/local/bin/lambda-agent >/dev/console 2>&1
+status=$?
+echo "[run-agent] lambda-agent exited with status $status" >/dev/console
+poweroff -f

cmd/lambda-agent/link_none.go

@@ -0,0 +1,18 @@
+//go:build !lambdatestlink
+
+package main
+
+import (
+	"errors"
+	"io"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+)
+
+// errNoHostTransport is returned until an authenticated host transport exists:
+// a normal build accepts no host work.
+var errNoHostTransport = errors.New("no authenticated host transport in this build")
+
+func openHostLink() (hostlink.Link, io.Closer, error) {
+	return nil, nil, errNoHostTransport
+}

cmd/lambda-agent/link_testlink.go

@@ -0,0 +1,27 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"errors"
+	"flag"
+	"io"
+	"os"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+)
+
+// The test carrier is unauthenticated and exists only in lambdatestlink builds
+// used for local proofs; it is not the guest protocol.
+var linkPath = flag.String("link", os.Getenv("LAMBDA_AGENT_LINK"), "test carrier device path")
+
+func openHostLink() (hostlink.Link, io.Closer, error) {
+	if *linkPath == "" {
+		return nil, nil, errors.New("no test carrier configured")
+	}
+	dev, err := os.OpenFile(*linkPath, os.O_RDWR, 0)
+	if err != nil {
+		return nil, nil, err
+	}
+	return hostlink.NewTestStream(dev), dev, nil
+}

cmd/lambda-agent/main.go

@@ -0,0 +1,87 @@
+// lambda-agent runs inside a Lambda guest. It serves the Lambda Runtime API on
+// guest loopback, starts the function's bootstrap, and relays invocations
+// between the host link and that runtime.
+//
+// A normal build has no host transport and refuses to start; the unauthenticated
+// test carrier exists only in lambdatestlink builds. The agent never listens for
+// the host.
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"syscall"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+)
+
+const (
+	runtimeAddr = "127.0.0.1:9001"
+	taskRoot    = "/var/task"
+)
+
+func main() { os.Exit(run()) }
+
+func run() int {
+	handler := flag.String("handler", os.Getenv("_HANDLER"), "function handler passed to the bootstrap")
+	flag.Parse()
+
+	link, closer, err := openHostLink()
+	if err != nil {
+		slog.Error("no host link; refusing to start", "err", err)
+		return 2
+	}
+	defer closer.Close()
+
+	srv := runtimeapi.New()
+	ln, err := net.Listen("tcp", runtimeAddr)
+	if err != nil {
+		slog.Error("listen runtime api", "err", err)
+		return 1
+	}
+	hs := &http.Server{Handler: srv.Handler(), ReadHeaderTimeout: 10 * time.Second}
+	go func() { _ = hs.Serve(ln) }()
+	defer hs.Close()
+
+	ctx, cancel := context.WithCancelCause(context.Background())
+	defer cancel(nil)
+
+	cmd := exec.CommandContext(ctx, taskRoot+"/bootstrap")
+	cmd.Env = []string{
+		"AWS_LAMBDA_RUNTIME_API=" + runtimeAddr,
+		"_HANDLER=" + *handler,
+		"LAMBDA_TASK_ROOT=" + taskRoot,
+		"PATH=/bin:/sbin:/usr/bin",
+	}
+	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+	cmd.Cancel = func() error { return cmd.Process.Signal(syscall.SIGTERM) }
+	cmd.WaitDelay = 5 * time.Second
+	if err := cmd.Start(); err != nil {
+		slog.Error("start bootstrap", "err", err)
+		return 1
+	}
+	exited := make(chan struct{})
+	go func() {
+		defer close(exited)
+		err := cmd.Wait()
+		cancel(fmt.Errorf("bootstrap exited: %w", err))
+	}()
+
+	err = agent.Run(ctx, link, srv)
+	cancel(errors.New("agent stopping"))
+	<-exited
+	if err != nil {
+		slog.Error("agent stopped", "err", err)
+		return 1
+	}
+	return 0
+}

contracts/ecs/v1/messages.go

@@ -126,8 +126,8 @@ type Assign struct {
 	ENIPrivateIP  string `json:"eniPrivateIp,omitempty"`
 	ENISubnetID   string `json:"eniSubnetId,omitempty"`
 	// TaskRoleARN is the task's IAM role (from the taskdef); when set, the agent
-	// serves its credentials at 169.254.170.2 under CredID, defaulting CredID to
-	// the taskID when the scheduler leaves it empty.
+	// serves its credentials at 169.254.170.2 under CredID, a scheduler-minted
+	// secret. An assign without a CredID gets no credentials.
 	CredID      string `json:"credId,omitempty"`
 	TaskRoleARN string `json:"taskRoleArn,omitempty"`
 	// ExecutionRoleARN is the task's execution role (from the taskdef); when set,

contracts/lambda/v1/README.md

@@ -0,0 +1,62 @@
+# Lambda v1 contracts
+
+This package will own the Lambda process contracts declared by ADR-0009:S13.
+It holds the L-GUEST declaration and, in `guest.go`, the connection vocabulary both ends share: the scope, the URI SAN roles, the ALPN identifier and the fw_cfg entry names. The transport and authentication are decided (ADR-0009:S17); the invocation message types and framing are not chosen, so no message types are defined here.
+It is not an AWS tenant-facing API.
+
+## L-GUEST: node executor to guest agent
+
+**Owner:** the Lambda domain.
+
+**Producers and consumers:** the node-local executor (`runtime/compute/lambda`) and the in-guest Lambda agent (`agents/lambda`), in both directions.
+
+**Operations:** bootstrap with scoped material, readiness, invoke, completion, bounded logs, shutdown.
+
+**Transport:** TCP over the microVM's network (ADR-0009:S17).
+The agent dials one connection to a listener the executor owns for the environment, and the guest exposes no listener for host control.
+At most one authenticated control connection is active per environment, and the agent may reconnect; how a prior connection and its in-flight work are handled is open (ADR-0009 M1).
+The network is shared and untrusted: no source address, port or bridge membership is identity or authority.
+Tenant code may be able to reach the listener, but it cannot open the agent's connection or key material, so any connection it makes fails authentication.
+
+**Authentication:** mutual TLS 1.3, bound to the environment ID and realization generation, using material the user runtime cannot read (ADR-0009:S17).
+For each generation the executor creates ephemeral self-signed host and guest certificates whose single URI SAN is `spinifex-lambda://<Region>/<account>/<function ID>/<environment ID>/<generation>/<role>`, with role `host` or `guest`.
+Each end validates the peer certificate against the one pinned peer certificate and separately requires that URI SAN with the opposite role; hostname verification is not relied on.
+ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake.
+Under TLS 1.3 the guest's handshake can complete before the host refuses its certificate, so the guest treats the connection as accepted only after the host's first message.
+
+**Bootstrap material:** the executor passes these fw_cfg entries under `opt/spinifex/lambda/`, and the agent reads them as root and unloads `qemu_fw_cfg` before user code starts.
+
+| Entry | Content |
+|---|---|
+| `guest-key` | Guest private key, PEM |
+| `guest-cert` | Guest certificate, PEM |
+| `host-cert` | The one pinned host certificate, PEM |
+| `expect` | `<Region>/<account>/<function ID>/<environment ID>/<generation>` |
+| `host-addr` | Host listener as `host:port` |
+
+**Authority:** one environment's attempt only.
+The user runtime never holds these keys, and sharing the guest with the agent gives user code no platform authority.
+
+**Idempotency and fencing:** an invoke carries the attempt-grant identity and the attempt ID.
+The agent accepts a run command only under a current grant, and a completion under a superseded grant is discarded and reported.
+How a grant is enforced and proven ended is open (ADR-0009 M1).
+
+**Errors and versions:** the agent and executor fail closed on an unknown protocol version or an identity mismatch.
+
+**Readiness:** environment readiness needs authenticated, current-generation agent evidence; a running hypervisor process does not count.
+
+**Payloads:** the contract must carry a synchronous payload and response of up to 6 MB and an asynchronous payload of up to 1 MB, plus framing, over a path whose size bound is declared.
+How payloads move is open (ADR-0009 M3).
+
+### Compatibility boundary
+
+The agent ships inside the runtime image pinned by each published version, so it is outside the coordinated release train.
+The executor supports every L-GUEST major carried by a still-supported runtime image.
+Retiring an image follows a published runtime-deprecation lifecycle that blocks create and update before it blocks invoke.
+
+Evidence required before the first agent image ships: a function on an old image invoked by a new executor, and the reverse during rollback.
+
+## L-RUNTIME
+
+The guest agent serves the AWS Lambda Runtime API, version `2018-06-01`, to the user runtime at `AWS_LAMBDA_RUNTIME_API`.
+Spinifex does not version this AWS contract, and conformance is measured against AWS.

contracts/lambda/v1/guest.go

@@ -0,0 +1,54 @@
+// Package lambdav1 holds the Lambda process-contract vocabulary shared by the
+// node executor and the guest agent. See README.md for the declarations.
+package lambdav1
+
+import (
+	"fmt"
+	"strings"
+)
+
+// ALPNv1 names L-GUEST major version 1; a peer offering any other fails the handshake.
+const ALPNv1 = "spinifex-lambda-guest/1"
+
+// FwCfgPrefix prefixes every fw_cfg entry the executor passes the guest.
+const FwCfgPrefix = "opt/spinifex/lambda/"
+
+// fw_cfg entry names under FwCfgPrefix.
+const (
+	EntryGuestKey  = "guest-key"
+	EntryGuestCert = "guest-cert"
+	EntryHostCert  = "host-cert"
+	EntryScope     = "expect"
+	EntryHostAddr  = "host-addr"
+)
+
+// Roles named as the last URI SAN path segment.
+const (
+	RoleHost  = "host"
+	RoleGuest = "guest"
+)
+
+const uriScheme = "spinifex-lambda"
+
+// Scope names one environment generation:
+// Region/account/function ID/environment ID/generation.
+type Scope string
+
+// ParseScope accepts exactly five non-empty, slash-separated segments.
+func ParseScope(s string) (Scope, error) {
+	parts := strings.Split(s, "/")
+	if len(parts) != 5 {
+		return "", fmt.Errorf("lambdav1: scope %q: want 5 segments, got %d", s, len(parts))
+	}
+	for _, p := range parts {
+		if p == "" || strings.ContainsAny(p, " \t\r\n?#%") {
+			return "", fmt.Errorf("lambdav1: scope %q: empty or invalid segment", s)
+		}
+	}
+	return Scope(s), nil
+}
+
+// URI is the single URI SAN a certificate for role in this scope carries.
+func (s Scope) URI(role string) string {
+	return uriScheme + "://" + string(s) + "/" + role
+}

contracts/lambda/v1/guest_test.go

@@ -0,0 +1,20 @@
+package lambdav1
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseScope(t *testing.T) {
+	s, err := ParseScope("ap-southeast-2/000000000001/fn-1/env-1/gen-1")
+	require.NoError(t, err)
+	assert.Equal(t, "spinifex-lambda://ap-southeast-2/000000000001/fn-1/env-1/gen-1/host", s.URI(RoleHost))
+	for _, bad := range []string{
+		"", "a/b/c/d", "a/b/c/d/e/f", "a//c/d/e", "a/b/c/d/e/", "a/b/c/d/e?x", "a/b/c/d/e#x", "a b/b/c/d/e",
+	} {
+		_, err := ParseScope(bad)
+		assert.Error(t, err, bad)
+	}
+}

contracts/operator/v1/admin.go

@@ -74,3 +74,65 @@ type ListAccountsResponse struct {
 	Accounts []AccountSummary `json:"accounts"`
 	Count    int              `json:"count"`
 }
+
+// IAMQuiescence is the coordinator's recorded view of IAM quiescence for one
+// fence. Recording only: it admits nothing, and QuiescedScope stays empty until
+// a final inventory re-read exists to catch a late write.
+type IAMQuiescence struct {
+	OperationID   string            `json:"operationId"`
+	Fence         uint64            `json:"fence"`
+	Generation    uint64            `json:"generation"`
+	QuiescedScope []string          `json:"quiescedScope"`
+	Unresolved    []UnresolvedScope `json:"unresolved"`
+	Consumers     []ConsumerBasis   `json:"consumers"`
+	Writers       []AccountedWriter `json:"writers"`
+	Inventory     []InventoryStatus `json:"inventory"`
+	EvaluatedAt   string            `json:"evaluatedAt"`
+}
+
+// UnresolvedScope names a scope or writer that blocks quiescence, with a fixed
+// reason. It is never a timeout.
+type UnresolvedScope struct {
+	Scope  string `json:"scope"`
+	Class  string `json:"class"`
+	Reason string `json:"reason"`
+}
+
+// ConsumerBasis cites an enforcement consumer and the accepted contract that
+// lets it count as having applied the generation.
+type ConsumerBasis struct {
+	Consumer string `json:"consumer"`
+	Basis    string `json:"basis"`
+}
+
+// AccountedWriter is a writer that cannot be fenced and is covered by the
+// owner's inventory instead. Ref names the tracked work that closes the gap.
+type AccountedWriter struct {
+	Writer string `json:"writer"`
+	Basis  string `json:"basis"`
+	Ref    string `json:"ref"`
+}
+
+// InventoryStatus is the owner inventory's status for one kind at evaluation.
+type InventoryStatus struct {
+	Kind   string `json:"kind"`
+	Status string `json:"status"`
+}
+
+// Classes of an UnresolvedScope.
+const (
+	UnresolvedClassFence    = "fence"
+	UnresolvedClassConsumer = "consumer"
+	UnresolvedClassWriter   = "writer"
+)
+
+// Fixed reasons of an UnresolvedScope.
+const (
+	ReasonFenceMismatch                = "fence-mismatch"
+	ReasonIAMReadbackFailed            = "iam-readback-failed"
+	ReasonNoAcceptedEnforcement        = "no-accepted-enforcement-contract"
+	ReasonNoCompleteOwnerInventory     = "no-complete-owner-inventory"
+	ReasonParticipantNotDeclared       = "participant-not-declared"
+	ReasonOwnerInventoryIncomplete     = "owner-inventory-incomplete"
+	ReasonFinalInventoryRereadNotWired = "final-inventory-reread-not-wired"
+)

docs/COMMANDS.md

@@ -152,12 +152,12 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `spx admin account create` | `--name` (required) | Connects to NATS → refuses if an account already holds the name (case- and whitespace-insensitive) → reserves the name in `spinifex-account-names` KV → CAS loop on `spinifex-account-counter:next_id` for sequential 12-digit ID → creates Account record in `spinifex-accounts` KV → creates default VPC → creates `admin` user, AdministratorAccess policy (Action:*, Resource:*) and one access key → prints credentials and writes a `spinifex-<name>` AWS CLI profile |
 | `spx admin account create --remote` | `--name` (required), `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA), `--client-token` (default: generated), `--source` (default: `spx-cli`) | Same account, created through `POST /admin/CreateAccount` over SigV4 instead of NATS — the path the public signup form uses, so a shell can exercise it. Credentials come from the standard AWS chain (env vars or `AWS_PROFILE`), never from the cluster master key; the caller must be an IAM user in the super-admin account holding `spinifex:CreateAccount`. Reuse the printed `--client-token` to retry: a fresh token would create a second account. |
 | `spx admin account list` | — | Connects to NATS → IAMService.ListAccounts() → prints table with Account ID, Name, Status, Created |
-| `spx admin account list --remote` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Same listing read through `POST /admin/ListAccounts` over SigV4 instead of NATS, so an off-cluster operator or the load-test harness sees its tenants without SSH access to a node. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:ListAccounts`. |
+| `spx admin account list --remote` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Same listing read through `POST /admin/ListAccounts` over SigV4 instead of NATS, so an off-cluster operator or the load-test harness sees its tenants without SSH access to a node. Credentials come from the standard AWS credential chain: on a cluster node that means `AWS_PROFILE=spinifex` (there is no default profile, so without it the command fails to find credentials), or equivalent credentials; the caller must be an IAM user in the super-admin account holding `spinifex:ListAccounts`. |
 | `spx admin account quota get <account-id>` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Reads the limits in force for one account through `POST /admin/GetAccountQuota` over SigV4, and reports for each dimension whether it came from the `[quota]` block in `awsgw.toml` (`config`) or from an override stored against the account (`override`). Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:GetAccountQuota`. |
 | `spx admin account quota set <account-id>` | `--vcpus`, `--vpcs`, `--subnets`, `--eips`, `--volumes`, `--volumes-gib`, `--rds-instances`, `--load-balancers`, `--clear`, plus `--endpoint`, `--region`, `--ca-bundle` | Overrides individual limits for one account through `POST /admin/PutAccountQuota`, leaving every dimension not named inheriting the configured baseline. `-1` uncaps a dimension; `0` is a real limit that denies every request on it. `--clear` removes the whole override record and cannot be combined with a limit flag. The account must already exist. Prints the resulting limits, so a caller never has to guess what it applied. The caller must hold `spinifex:PutAccountQuota`. |
-| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record) | Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → seals the account's quota override record so no later write can recreate it, which needs every gateway node the cluster configures to answer a read-only probe first (otherwise nothing is sealed and the account stays `TERMINATING`) → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Platform-managed serving endpoints are reaped ahead of instances, because only their own delete releases the ENI and attached data volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
+| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record), `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Always runs through `POST /admin/DeleteAccount` over SigV4 and follows `DescribeAccountDeletion` to completion; the CLI never tears an account down in its own process, so a CLI delete and an API delete cannot race. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → seals the account's quota override record so no later write can recreate it, which needs every gateway node the cluster configures to answer a read-only probe first (otherwise nothing is sealed and the account stays `TERMINATING`) → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Platform-managed serving endpoints are reaped ahead of instances, because only their own delete releases the ENI and attached data volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
 | `spx admin account delete --force` | as above | Escalates only for resources the ordinary API refuses: it clears a volume attachment in the control plane without the guest's cooperation (`ec2.ForceDetachVolume`), hard-destroys after a graceful attempt times out, and treats an already-missing resource as deleted. It never reorders the stages and never widens scope beyond the account. Use it once the guest is being destroyed — this is the escape from a volume stuck attached to an instance that will not terminate, where neither can otherwise be deleted. |
-| `spx admin account delete --remote` | as above, plus `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Same teardown driven through `POST /admin/DeleteAccount` over SigV4 instead of NATS, then follows `DescribeAccountDeletion` to completion, printing each stage as it lands. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. |
+| `spx admin account delete --remote` | as above | Accepted for existing scripts and ignored: every delete already goes through the admin API. |
 | `spx admin principal create <name>` | `--grant` (repeatable or comma-separated; default: every admin method — `CreateAccount`, `DeleteAccount`, `DescribeAccountDeletion`, `GetAccountQuota`, `ListAccounts`, `PutAccountQuota`) | Creates an IAM user in the super-admin account with an inline `spx-admin-methods` policy allowing the requested methods, revokes any access key left from a previous run, and mints one new key. Each method is named rather than granted as `spinifex:*`, so a later addition to the admin surface is not authorised by an existing key. The principal's grants are exactly that one policy: re-running with fewer `--grant` values narrows it, and any other inline policy on the user is removed, so a rename cannot leave a wider grant behind. The secret is printed once; store it in a secret manager or an AWS profile, never in a config file. `admin` is refused as a name — it is the account's own administrator and rotating its key would revoke the bootstrap credential. |
 | `spx admin principal list` | — | Lists every principal in the super-admin account with its access-key count and what it may call. An attached policy is reported by name (`AdministratorAccess (attached)`) rather than expanded, which is how an unscoped principal is spotted. |
 | `spx admin principal revoke <name>` | — | Deletes every access key the named principal holds, leaving the user and its policy in place. Immediate and cluster-wide; re-issue with `principal create`. |

docs/admin/account-management/README.md

@@ -236,12 +236,12 @@ Each stage runs to empty before the next begins, so nothing is deleted while som
 A large account takes minutes to tear down. The API does not hold the connection open for it: `DeleteAccount` starts the job, returns a `deletionId` immediately, and progress is read from `DescribeAccountDeletion`.
 
 ```bash
-AWS_PROFILE=spinifex-operator spx admin account delete 000000000042 --remote \
+AWS_PROFILE=spinifex-operator spx admin account delete 000000000042 \
   --name customer@example.com --yes \
   --endpoint https://node1.example.com:9999 --region us-west-1
 ```
 
-`spx --remote` starts the job and then polls, printing each stage as it lands. Your own client can do the same, or start the job and check back later — the deletion record outlives the account, so `DescribeAccountDeletion` stays answerable after the account is gone.
+`spx admin account delete` always goes through the admin API, so it signs with the standard AWS credential chain. On a cluster node that means `AWS_PROFILE=spinifex` (there is no default profile, so without it the command cannot find credentials), or equivalent credentials for an IAM user holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`, and a reachable gateway. The older `--remote` flag is accepted and ignored. The command starts the job and then polls, printing each stage as it lands. Your own client can do the same, or start the job and check back later — the deletion record outlives the account, so `DescribeAccountDeletion` stays answerable after the account is gone.
 
 One job runs per account. A second `DeleteAccount` with a different token while one is running returns `OperationInProgress`; with the *same* token it returns the running job, which is what makes a retry safe.
 

docs/containers/eks/README.md

@@ -58,7 +58,7 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM
 **Spinifex specifics**
 
 - **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
-- **Security groups are auto-managed.** Spinifex creates the cluster and node-group security groups deterministically (e.g. `eks-cluster-<name>-nodegroup-sg`); `vpc_config.security_group_ids` is ignored. To expose a workload you add an ingress rule to the auto-managed node-group SG.
+- **The cluster security group is auto-managed.** As on EKS, Spinifex creates one `eks-cluster-sg-<name>-<id>` in your VPC, tagged `kubernetes.io/cluster/<name>=owned` and `aws:eks:cluster-name`, allowing all traffic from itself. It is attached to the workers and the private-endpoint network interface, and `DescribeCluster` reports it as `clusterSecurityGroupId`. A launch template that names security groups replaces it on the workers, as on AWS. Clusters created before this change keep their `-control-plane-sg` and `-nodegroup-sg` until you recreate them. Deleting a cluster deletes the cluster security group and removes only the rules on your other security groups that reference it. `vpc_config.security_group_ids` is ignored.
 - **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
 - **Workers launch with IMDSv2 required and an IMDS hop limit of 2.**
   This matches AWS managed node groups and lets a pod without IRSA reach the node role through the SDK default credential chain.
@@ -143,6 +143,25 @@ No user-facing operation replaces a control plane, and automatic replacement doe
 Recreate the cluster to adopt it.
 User-workload injection through a cluster created with an older image is not measured.
 
+### Add-on versions
+
+Add-on versions use the AWS form `v<x.y.z>-eksbuild.<n>`, for example `v1.40.1-eksbuild.1`.
+`DescribeAddonVersions` returns versions in this form, newest first, and every other add-on response reports the version the same way.
+Spinifex ships one EKS build of each upstream version, so `<n>` is `1` today.
+
+`DescribeAddonVersions` honours `addonName`, `kubernetesVersion`, `types`, `publishers`, `owners`, `maxResults` and `nextToken`.
+Each version carries `compatibilities` with the `clusterVersion`, the `platformVersions`, and a `defaultVersion` flag that is true for exactly one version per add-on and cluster version.
+A `kubernetesVersion` that Spinifex does not run returns no add-ons.
+
+`CreateAddon` and `UpdateAddon` accept the AWS form of a version in the catalogue.
+They also accept the bare upstream form, such as `1.40.1`, and Spinifex logs each use at Info.
+This is a Spinifex leniency that AWS does not offer, and it exists so that older scripts keep working.
+Any other value is refused with a message that names `addonVersion` and lists the supported versions, for example `v1.40.1-eksbuild.2`, `1.40` or `v1.40.1`.
+
+In Terraform, pass the AWS form to `aws_eks_addon.addon_version`, either as a literal or from `data.aws_eks_addon_version`.
+The AWS provider validates the format before it calls the API, so it rejects the bare form.
+Leaving `addon_version` unset still installs the catalogue default.
+
 ## Prerequisites
 
 > [!IMPORTANT]
@@ -225,12 +244,11 @@ The list shows the node group workers only. The control-plane VMs never appear a
 
 ### 4. Expose a workload
 
-Workers join an auto-managed security group that only allows intra-cluster traffic. To reach a NodePort service, add one ingress rule to that SG (look it up by its deterministic name):
+Workers join the cluster security group, which only allows intra-cluster traffic. To reach a NodePort service, add one ingress rule to it (`DescribeCluster` reports its id):
 
 ```bash
-SG=$(aws ec2 describe-security-groups \
-  --filters Name=group-name,Values=eks-cluster-demo-nodegroup-sg \
-  --query 'SecurityGroups[0].GroupId' --output text)
+SG=$(aws eks describe-cluster --name demo \
+  --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text)
 
 aws ec2 authorize-security-group-ingress \
   --group-id "$SG" --protocol tcp --port 30080 --cidr 0.0.0.0/0
@@ -272,7 +290,7 @@ The form notes that the authentication mode is API and that the legacy aws-auth
 
 - The **Access** tab lists IAM principals and their cluster-access policies — add an access entry to let another user or role reach the cluster.
 - The **Addons** tab installs and shows the health of managed addons.
-- The **Networking** tab shows the cluster's VPC, subnets, endpoint access, and the auto-managed security groups.
+- The **Networking** tab shows the cluster's VPC, subnets, endpoint access, and the auto-managed cluster security group.
 
 To run `kubectl` against the cluster, follow the AWS CLI tab's `update-kubeconfig` step.
 
@@ -355,11 +373,11 @@ Control plane readiness and kubectl authentication above lists which clusters re
 - Confirm the workers can pull images — a default route to an Internet Gateway (public) or NAT Gateway (private).
 - Confirm the node role has `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, and `AmazonEC2ContainerRegistryReadOnly`.
 
-**Workload unreachable from outside the cluster.** The node-group security group only allows intra-cluster traffic by default. Add an ingress rule for your NodePort to `eks-cluster-<name>-nodegroup-sg`, as shown in the CLI tab.
+**Workload unreachable from outside the cluster.** The cluster security group only allows intra-cluster traffic by default. Add an ingress rule for your NodePort to it, as shown in the CLI tab. An Ingress served by the AWS Load Balancer Controller needs no rule: the controller adds it to the cluster security group.
 
 **`create-cluster` rejected with an authentication-mode error.** `CONFIG_MAP` is the one mode Spinifex rejects, because it has no `aws-auth` ConfigMap path. Set `authentication_mode` to `API` or `API_AND_CONFIG_MAP`, or leave it unset, and grant access with access entries.
 
-**Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, leave `addon_version` unset so Spinifex selects the catalog default.
+**Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, use the AWS form from `DescribeAddonVersions` (see Add-on versions above), or leave `addon_version` unset so Spinifex selects the catalog default.
 
 ## Reference Architectures
 

docs/coverage/README.md

@@ -17,7 +17,7 @@ tags:
 
 ## Overview
 
-The platform serves **405 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
+The platform serves **406 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
 
 | Service | Operations |
 |---|---:|
@@ -27,9 +27,9 @@ The platform serves **405 operations** across the AWS APIs below. Every page nam
 | [ECS](/coverage/ecs) | 31 |
 | [EKS](/coverage/eks) | 27 |
 | [ELBv2](/coverage/elbv2) | 36 |
-| [IAM](/coverage/iam) | 81 |
+| [IAM](/coverage/iam) | 82 |
 | [RDS](/coverage/rds) | 27 |
 | [Route 53](/coverage/route53) | 13 |
 | [S3](/coverage/s3) | 26 |
 | [STS](/coverage/sts) | 5 |
-| **Total** | **405** |
+| **Total** | **406** |

docs/coverage/coverage.json

@@ -2938,7 +2938,8 @@
           "UpdateAccessKey",
           "UpdateAssumeRolePolicy",
           "UpdateRole",
-          "UpdateRoleDescription"
+          "UpdateRoleDescription",
+          "UpdateUser"
         ],
         "implemented": [
           "AddRoleToInstanceProfile",
@@ -3021,7 +3022,8 @@
           "UpdateAccessKey",
           "UpdateAssumeRolePolicy",
           "UpdateRole",
-          "UpdateRoleDescription"
+          "UpdateRoleDescription",
+          "UpdateUser"
         ],
         "stubbed": [],
         "unsupported": [],
@@ -3121,7 +3123,6 @@
           "UpdateServerCertificate",
           "UpdateServiceSpecificCredential",
           "UpdateSigningCertificate",
-          "UpdateUser",
           "UploadSSHPublicKey",
           "UploadServerCertificate",
           "UploadSigningCertificate"

docs/coverage/iam/README.md

@@ -18,7 +18,7 @@ tags:
 
 ## Overview
 
-Spinifex implements **81 operations** in the IAM `2010-05-08` API model.
+Spinifex implements **82 operations** in the IAM `2010-05-08` API model.
 
 ### Scope
 
@@ -109,3 +109,4 @@ All IAM operations are account-scoped. The root user of an account bypasses poli
 | `UpdateAssumeRolePolicy` |
 | `UpdateRole` |
 | `UpdateRoleDescription` |
+| `UpdateUser` |

docs/coverage/internal-report.md

@@ -987,7 +987,7 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 
 ## iam
 
-`2010-05-08` — 81 of 180 modelled operations implemented (45.0%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
+`2010-05-08` — 82 of 180 modelled operations implemented (45.6%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
 
 ### Not implemented
 
@@ -1069,7 +1069,6 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 - `UpdateOpenIDConnectProviderThumbprint`
 - `UpdateSAMLProvider`
 - `UpdateServerCertificate`
-- `UpdateUser`
 - `UploadServerCertificate`
 
 ### Not applicable — do not build

docs/package-boundary/eks-addon-lifecycle.md

@@ -20,7 +20,7 @@ There is no generation, so the record mixes desired fields with the latest obser
 | Durable record | `clusters/{cluster}/addons/{addon}` holds `Record` (`domains/eks/addon/record.go`): `addonName`, `addonVersion`, `status`, `serviceAccountRoleArn`, `configurationValues`, `health`, `arn`, `tags`, `createdAt`, `modifiedAt` (the optional ones `omitempty`). `clusters/{cluster}/addons/{addon}/manifest` holds `Manifest` (`domains/eks/addon/record.go`): `addonName`, `addonVersion`, `serviceAccountRoleArn`, `configurationValues`. Listing skips the manifest sub-key by segment count. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestAddonRecord_DecodesHandWrittenJSON`, `domains/eks/addon/store_test.go` `TestRecord_RoundTrip`, `TestList_SkipsManifestSubKeys` |
 | Writers | Record: the owner in `domains/eks/addon/owner.go`, reached from the `CreateAddon`, `UpdateAddon` and `DeleteAddon` handlers, the cluster reconciler's report relay and the GPU node group launch: `Owner.Create` (unconditional put), `Owner.Update` and its failure marking (revision CAS), `Owner.Delete` (get then delete), `Owner.ApplyReport` (revision CAS), `Owner.EnsureGPUDevicePlugin` through `Owner.Create`; and the cluster prefix purge (`DeleteClusterChildren`). Manifest: `StagingInstaller.Install` (unconditional put on create and update), `StagingInstaller.Uninstall` (`domains/eks/addon/delivery.go`), `Owner.Delete` and the prefix purge. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestUpdateAddon_ReportsSuccessfulWhileRecordIsUpdating`, `TestDeleteAddon_ReturnsDeletingAfterErasingRecordAndManifest`, `TestDeleteClusterChildren_SweepsAddonRecordsAndManifestsScopedToCluster`, `domains/eks/addon/store_test.go` `TestCasUpdate`, `TestDeleteRecord_RemovesRecordAndManifest` |
 | Readers | `ListAddons`, `DescribeAddon`, `CreateAddon` (existence check) and the reconciler read the record. `ListStagedAddonManifests` reads every `/manifest` key for the cluster (`addon.ListManifests`); it is served on `eks.ListStagedAddonManifests` (`daemon/daemon.go`) and reached only through the gateway's internal-addons route. | `handlers/eks/addons_test.go` `TestAddon_DescribeListDelete`, `TestListStagedAddonManifests` |
-| Catalogue | Static in-binary catalogue (`domains/eks/addon/catalog.go`): `aws-load-balancer-controller`, `argocd`, `aws-ebs-csi-driver`, and hidden-but-creatable `nvidia-device-plugin` and `spinifex-noop`; one version each, the first being the default. Create and update reject unknown names and versions with `InvalidParameterValue`. | `domains/eks/addon/catalog_test.go`, `handlers/eks/addons_test.go` `TestDescribeAddonVersions_ReturnsCatalog`, `TestCreateAddon_UnknownAddonRejected`, `TestCreateAddon_UnknownVersionRejected`, `TestUpdateAddon_UnknownVersionRejected` |
+| Catalogue | Static in-binary catalogue (`domains/eks/addon/catalog.go`): `aws-load-balancer-controller`, `argocd`, `aws-ebs-csi-driver`, and hidden-but-creatable `nvidia-device-plugin` and `spinifex-noop`; one version each, the first being the default. Versions are stored and used internally in the bare upstream form, and API responses show the AWS form `v<x.y.z>-eksbuild.<n>` (`Spec.AWSVersion`). Create and update accept the AWS form or the exact bare form and reject unknown names and other version strings with `InvalidParameterValue`, which the gateway returns as `InvalidParameterException`. | `domains/eks/addon/catalog_test.go`, `handlers/eks/addons_test.go` `TestDescribeAddonVersions_ReturnsCatalog`, `TestCreateAddon_UnknownAddonRejected`, `TestCreateAddon_UnknownVersionRejected`, `TestUpdateAddon_UnknownVersionRejected` |
 | Desired state | Version, `serviceAccountRoleArn`, `configurationValues` and create-time tags. `resolveConflicts`, `podIdentityAssociations` and `clientRequestToken` are accepted and dropped; `preserve` on delete is ignored. `configurationValues` reaches the guest but the agent discards it (`mulga-eks-addon-sync.sh` `sync_once`), so it has no effect. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestDeleteAddon_ReturnsDeletingAfterErasingRecordAndManifest` |
 | Observed state | `status` and `health` are overwritten by guest reports on the same record as the desired fields. `health` is the latest failure message, projected as one `AddonIssue` with only `message`. Statuses held: `CREATING`, `ACTIVE`, `UPDATING`, `DEGRADED`, `CREATE_FAILED`; `DELETING` is defined but never stored. | `TestAddonRecord_DecodesHandWrittenJSON`, `handlers/eks/addon_status_reconciler_test.go` `TestApplyAddonStatusReport` |
 | Generation | None. Reports carry `version` and `ts`, and neither is read: any report for the add-on name applies to whatever is currently desired. | `TestAddonStatusReport_MismatchedVersionIsAccepted` |

docs/terraform-workbooks/eks-gitops-argocd/README.md

@@ -79,7 +79,8 @@ Terraform manages the cluster, the addons, the ACM cert, and the HTTPS Ingress;
 - The `argocd` and `aws-ebs-csi-driver` bundles must be baked into the `eks-node` AMI — `describe-addon` returns `no baked bundle` otherwise.
 - The EBS-CSI default StorageClass uses `provisioner: ebs.csi.aws.com`; a PVC against it provisions a **Viperblock-backed EBS volume**.
 - If the PVC stays `Pending`, the app still serves (the counter falls back to in-memory) — see [Troubleshooting](#troubleshooting).
-- Leave addon versions unset (the AWS provider and the catalog disagree on the version-string format).
+- Addon versions are unset, so Spinifex installs each catalog default.
+  To pin one, use the AWS form that `DescribeAddonVersions` returns, such as `v3.0.23-eksbuild.1`.
 - The Argo CD `Application` is a `kubernetes_manifest`, so the `argoproj.io` CRDs must exist before `apply` — apply the parent and let the addon reach `ACTIVE` first.
 
 **Prerequisites:**

docs/terraform-workbooks/eks-gitops-argocd/main.tf

@@ -471,8 +471,7 @@ resource "aws_eks_cluster" "this" {
   # expects: access_config reads as absent (and its ForceNew
   # bootstrap_cluster_creator_admin_permissions would replace the live cluster on
   # re-add), and vpc_config.security_group_ids reads back as drift. Ignore both so
-  # later applies (e.g. adding the Argo CD NodePort rule) neither replace the
-  # cluster nor detach its security groups.
+  # later applies neither replace the cluster nor detach its security groups.
   lifecycle {
     ignore_changes = [access_config, vpc_config[0].security_group_ids]
   }
@@ -520,8 +519,8 @@ resource "aws_eks_node_group" "workers" {
 #
 # Spinifex wires the controller's AWS credentials and ELB-eligible subnets at the
 # node level, so no IRSA role or subnet tagging is needed here. addon_version is
-# omitted: the AWS provider demands a v-prefixed version that the catalog rejects,
-# so let Spinifex default to its catalog version.
+# omitted so Spinifex installs its catalog default. To pin one, use the AWS form
+# from DescribeAddonVersions, for example v2.11.0-eksbuild.1.
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "lbc" {
@@ -542,7 +541,7 @@ resource "aws_eks_addon" "lbc" {
 # Installed through the EKS API; Spinifex stages the bundle host-side and the
 # worker renders it into the K3s auto-deploy dir. The workloads module registers
 # the git repo and creates the Argo CD Application that syncs the demo app.
-# addon_version omitted (see the LBC note).
+# addon_version omitted: the catalog default is installed (see the LBC note).
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "argocd" {
@@ -562,7 +561,7 @@ resource "aws_eks_addon" "argocd" {
 #
 # Ships a default gp3 StorageClass (provisioner ebs.csi.aws.com). A PVC in the
 # demo app's git manifests dynamically provisions a Viperblock-backed EBS volume
-# so the app's state survives pod restarts. addon_version omitted (see LBC note).
+# so the app's state survives pod restarts. addon_version omitted: catalog default.
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "ebs_csi" {
@@ -642,53 +641,6 @@ resource "aws_acm_certificate" "ingress" {
   }
 }
 
-# ---------------------------------------------------------------------------
-# Let the ALB reach the workers' NodePort
-#
-# Spinifex auto-manages the nodegroup SG and admits only intra-cluster traffic.
-# The LBC-provisioned ALB lives in this VPC, so admit the VPC CIDR on the
-# NodePort. Look the SG up by its deterministic name and add one rule.
-# ---------------------------------------------------------------------------
-
-data "aws_security_group" "nodegroup" {
-  filter {
-    name   = "group-name"
-    values = ["eks-cluster-${var.cluster_name}-nodegroup-sg"]
-  }
-
-  filter {
-    name   = "vpc-id"
-    values = [aws_vpc.main.id]
-  }
-
-  depends_on = [aws_eks_node_group.workers]
-}
-
-resource "aws_vpc_security_group_ingress_rule" "nodeport_from_vpc" {
-  security_group_id = data.aws_security_group.nodegroup.id
-  cidr_ipv4         = aws_vpc.main.cidr_block
-  from_port         = var.node_port
-  to_port           = var.node_port
-  ip_protocol       = "tcp"
-
-  tags = {
-    Name = "${var.cluster_name}-alb-nodeport"
-  }
-}
-
-# Same, for the Argo CD UI NodePort its ALB forwards to.
-resource "aws_vpc_security_group_ingress_rule" "argocd_nodeport_from_vpc" {
-  security_group_id = data.aws_security_group.nodegroup.id
-  cidr_ipv4         = aws_vpc.main.cidr_block
-  from_port         = var.argocd_node_port
-  to_port           = var.argocd_node_port
-  ip_protocol       = "tcp"
-
-  tags = {
-    Name = "${var.cluster_name}-argocd-nodeport"
-  }
-}
-
 # ---------------------------------------------------------------------------
 # Outputs
 # ---------------------------------------------------------------------------

docs/terraform-workbooks/eks-https-ingress/README.md

@@ -64,7 +64,6 @@ The LBC is installed as a managed **addon** (`aws_eks_addon`). Spinifex wires th
 | Node Group | `workers` | `node_desired_size` `t3.large` worker(s) — 1 or 3 |
 | Addon | `aws-load-balancer-controller` | Provisions the ALB from the Ingress |
 | ACM Certificate | `eks-https-ingress` | Self-signed, imported; attached to the HTTPS listener |
-| SG Ingress Rule | `eks-https-alb-nodeport` | Admits the VPC CIDR to the workers' NodePort |
 | K8s Deployment + Service | `spinifex-demo` | Themed demo image, 2 replicas, NodePort |
 | K8s Ingress | `spinifex-demo` | `ingressClassName: alb`, HTTPS via the ACM cert |
 
@@ -72,9 +71,10 @@ The LBC is installed as a managed **addon** (`aws_eks_addon`). Spinifex wires th
 
 - **The LBC needs no IRSA or subnet tagging here.** Credentials and ELB-eligible subnets are injected by Spinifex at the node level and into the `alb` `IngressClassParams`. The Ingress just sets `ingressClassName: alb` plus the `alb.ingress.kubernetes.io/*` annotations.
 - **`spinifex.io/managed-ingress = "false"`** disables the K3s built-in traefik/servicelb so the LBC owns ingress, matching AWS parity. Omit the tag (or set it `true`) and the built-in ingress stays on.
-- The ALB targets the workers' **NodePort** (`target-type: instance`). The one SG rule admits the VPC CIDR on that port so the in-VPC ALB can reach the workers; the worker SG is otherwise auto-managed.
+- The ALB targets the workers' **NodePort** (`target-type: instance`). The AWS Load Balancer Controller adds the ALB-to-NodePort rule to the cluster security group itself, so the workbook creates no security group rule.
 - Spinifex ACM supports **ImportCertificate only** — the cert is generated by the `tls` provider and imported, not requested.
-- **Leave `addon_version` unset** (see the [quickstart/addons notes](/docs/eks-quickstart)) — the AWS provider and the Spinifex catalog disagree on the version string format, so let the server choose.
+- **`addon_version` is unset**, so Spinifex installs its catalog default.
+  To pin a version, use the AWS form that `DescribeAddonVersions` returns, such as `v2.11.0-eksbuild.1`, or read it from `data.aws_eks_addon_version`.
 
 **Prerequisites:**
 
@@ -179,16 +179,17 @@ Confirm the addon is `ACTIVE` and the cluster carries `spinifex.io/managed-ingre
 
 ### HTTPS Loads but Returns 502 / 504
 
-The ALB reached the workers but the backend didn't answer. Check the demo pods and the NodePort SG rule:
+The ALB reached the workers but the backend didn't answer. Check the demo pods and the rule the controller added to the cluster security group:
 
 ```bash
 kubectl get pods -o wide
 kubectl get svc spinifex-demo
-aws ec2 describe-security-groups --filters "Name=group-name,Values=eks-cluster-eks-https-nodegroup-sg" \
+SG=$(aws eks describe-cluster --name eks-https --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text)
+aws ec2 describe-security-groups --group-ids "$SG" \
   --query 'SecurityGroups[0].IpPermissions'
 ```
 
-The `eks-https-alb-nodeport` rule must admit the VPC CIDR on the NodePort, and the target group health check (`/healthz`) must pass.
+The cluster security group must admit the load balancer's security group on the NodePort, and the target group health check (`/healthz`) must pass.
 
 ### Addon Fails With "no baked bundle"
 

docs/terraform-workbooks/eks-https-ingress/main.tf

@@ -16,7 +16,8 @@
 # The workloads/ module then creates a Deployment, a NodePort Service, and an
 # Ingress (ingressClassName: alb). The LBC reconciles that Ingress into an
 # internet-facing ALB that terminates TLS with the ACM cert and forwards to the
-# workers' NodePort. The cluster's ELB-eligible subnets are injected by Spinifex
+# workers' NodePort, adding the ALB-to-NodePort rule to the cluster security group
+# itself. The cluster's ELB-eligible subnets are injected by Spinifex
 # into the alb IngressClassParams, so the Ingress needs no subnet annotations.
 #
 # Usage:
@@ -446,8 +447,8 @@ resource "aws_eks_node_group" "workers" {
 #
 # Spinifex wires the controller's AWS credentials and ELB-eligible subnets at the
 # node level, so no IRSA role or subnet tagging is needed here. addon_version is
-# omitted: the AWS provider demands a v-prefixed version that the catalog rejects,
-# so let Spinifex default to its catalog version.
+# omitted so Spinifex installs its catalog default. To pin one, use the AWS form
+# from DescribeAddonVersions, for example v2.11.0-eksbuild.1.
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "lbc" {
@@ -503,40 +504,6 @@ resource "aws_acm_certificate" "ingress" {
   }
 }
 
-# ---------------------------------------------------------------------------
-# Let the ALB reach the workers' NodePort
-#
-# Spinifex auto-manages the nodegroup SG and admits only intra-cluster traffic.
-# The LBC-provisioned ALB lives in this VPC, so admit the VPC CIDR on the
-# NodePort. Look the SG up by its deterministic name and add one rule.
-# ---------------------------------------------------------------------------
-
-data "aws_security_group" "nodegroup" {
-  filter {
-    name   = "group-name"
-    values = ["eks-cluster-${var.cluster_name}-nodegroup-sg"]
-  }
-
-  filter {
-    name   = "vpc-id"
-    values = [aws_vpc.main.id]
-  }
-
-  depends_on = [aws_eks_node_group.workers]
-}
-
-resource "aws_vpc_security_group_ingress_rule" "nodeport_from_vpc" {
-  security_group_id = data.aws_security_group.nodegroup.id
-  cidr_ipv4         = aws_vpc.main.cidr_block
-  from_port         = var.node_port
-  to_port           = var.node_port
-  ip_protocol       = "tcp"
-
-  tags = {
-    Name = "${var.cluster_name}-alb-nodeport"
-  }
-}
-
 # ---------------------------------------------------------------------------
 # Outputs
 # ---------------------------------------------------------------------------

docs/terraform-workbooks/eks-quickstart/README.md

@@ -45,7 +45,7 @@ Under the hood it keeps things minimal: a VPC with two public subnets, an IAM ro
 - Creating the EKS cluster and worker IAM roles with faithful managed-policy attachments
 - Provisioning an `aws_eks_cluster` and a managed `aws_eks_node_group`
 - Pointing the Kubernetes provider at the new cluster and deploying a workload from the same `apply`
-- Opening one rule on the auto-managed worker security group to expose a NodePort
+- Opening one rule on the cluster security group to expose a NodePort
 
 **What gets created**
 
@@ -58,13 +58,13 @@ Under the hood it keeps things minimal: a VPC with two public subnets, an IAM ro
 | ECR Repository | `spinifex-demo` | Holds the demo image the workers pull |
 | EKS Cluster | `eks-quickstart` | Public API endpoint, API auth mode, Kubernetes 1.32 |
 | Node Group | `default` | `node_desired_size` `t3.medium` worker(s) — 1 or 3 |
-| SG Ingress Rule | `eks-quickstart-demo-nodeport` | Opens the demo NodePort on the auto-managed worker SG |
+| SG Ingress Rule | `eks-quickstart-demo-nodeport` | Opens the demo NodePort on the cluster security group |
 | K8s Deployment | `spinifex-demo` | The themed demo image from ECR, 2 replicas |
 | K8s Service | `spinifex-demo` | NodePort publishing the demo on the worker |
 
 **Spinifex specifics**
 
-- The cluster's security groups are **auto-managed** — `vpc_config.security_group_ids` is ignored. To reach a NodePort, this workbook looks the worker SG up by its deterministic name (`eks-cluster-<name>-nodegroup-sg`) and adds a single ingress rule.
+- The cluster security group is **auto-managed**, as on EKS: Spinifex creates `eks-cluster-sg-<name>-<id>`, attaches it to the workers, and reports it as `vpc_config[0].cluster_security_group_id`. `vpc_config.security_group_ids` is ignored. To reach a NodePort, this workbook adds a single ingress rule to that security group.
 - The worker AMI is always Spinifex's `eks-node` image; `ami_type` is recorded but does **not** select the image.
 - `authentication_mode` can be `"API"` or `"API_AND_CONFIG_MAP"`, and bare `"CONFIG_MAP"` is rejected. Access is granted through access entries, as there is no `aws-auth` ConfigMap path.
 - The workers need outbound internet (here via the IGW) to pull the demo container image.
@@ -182,8 +182,9 @@ aws ec2 describe-instances --filters "Name=tag:spinifex:eks-cluster,Values=eks-q
 # Did the demo pods roll out?
 kubectl get pods -o wide
 
-# Is the NodePort rule on the worker SG?
-aws ec2 describe-security-groups --filters "Name=group-name,Values=eks-cluster-eks-quickstart-nodegroup-sg" \
+# Is the NodePort rule on the cluster security group?
+SG=$(tofu output -raw cluster_security_group_id)
+aws ec2 describe-security-groups --group-ids "$SG" \
   --query 'SecurityGroups[0].IpPermissions'
 ```
 

docs/terraform-workbooks/eks-quickstart/main.tf

@@ -310,29 +310,15 @@ resource "aws_eks_node_group" "default" {
 }
 
 # ---------------------------------------------------------------------------
-# Open the NodePort on the auto-managed nodegroup SG
+# Open the NodePort on the cluster security group
 #
-# Spinifex creates the worker SG itself (vpc_config.security_group_ids is
-# ignored) and admits only intra-cluster traffic. To reach the demo NodePort
-# from a browser, look the SG up by its deterministic name and add one rule.
+# Spinifex creates the cluster security group itself, as EKS does, and attaches
+# it to the workers. It admits only intra-cluster traffic. To reach the demo
+# NodePort from a browser, add one rule to the SG the cluster reports.
 # ---------------------------------------------------------------------------
 
-data "aws_security_group" "nodegroup" {
-  filter {
-    name   = "group-name"
-    values = ["eks-cluster-${var.cluster_name}-nodegroup-sg"]
-  }
-
-  filter {
-    name   = "vpc-id"
-    values = [aws_vpc.main.id]
-  }
-
-  depends_on = [aws_eks_node_group.default]
-}
-
 resource "aws_vpc_security_group_ingress_rule" "nodeport" {
-  security_group_id = data.aws_security_group.nodegroup.id
+  security_group_id = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id
   cidr_ipv4         = var.browse_cidr
   from_port         = var.node_port
   to_port           = var.node_port
@@ -367,6 +353,10 @@ output "region" {
   value = var.region
 }
 
+output "cluster_security_group_id" {
+  value = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id
+}
+
 output "node_port" {
   value = var.node_port
 }

scripts/build-lambda-runtime-microvm-image.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+# build-lambda-runtime-microvm-image.sh — build the Lambda runtime microVM kernel
+# + initramfs: a guest running the Lambda guest agent, with virtio-net for its
+# host control connection. It is not shipped.
+#
+# Inputs:
+#   $REPO_ROOT/build/microvm/lambda-runtime/{init.sh,inittab,run-agent}
+#   $REPO_ROOT/bin/lambda-agent  (make bin/lambda-agent)
+#
+# Outputs:
+#   $REPO_ROOT/build/microvm/lambda-runtime/vmlinuz
+#   $REPO_ROOT/build/microvm/lambda-runtime/initramfs.cpio.gz
+#
+# Rootfs strategy and module stripping: scripts/lib/microvm-image.sh.
+# shellcheck disable=SC2034 # variables are consumed by the sourced library
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+BUILD_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
+INIT_SH="$BUILD_DIR/init.sh"
+INITTAB="$BUILD_DIR/inittab"
+RUN_AGENT="$BUILD_DIR/run-agent"
+LAMBDA_AGENT_BIN="$REPO_ROOT/bin/lambda-agent"
+TAG="[build-lambda-runtime-microvm-image]"
+
+ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
+ALPINE_MIRROR="${ALPINE_MIRROR:-https://dl-cdn.alpinelinux.org/alpine}"
+# kmod: the image's own depmod/modprobe rebuild module metadata. busybox ships ip.
+ALPINE_PACKAGES="busybox linux-virt kmod ca-certificates"
+# tiny_power_button turns the ACPI power button into a signal busybox init
+# handles as poweroff.
+MICROVM_EXTRA_MODULES="${MICROVM_EXTRA_MODULES:-tiny_power_button}"
+# virtio_net needs failover and net_failover, which the virtio* glob misses.
+MICROVM_EXTRA_KEEP_GLOBS=("*failover*.ko*")
+MICROVM_GATE_MODULES="qemu_fw_cfg virtio_net"
+
+# shellcheck source=lib/microvm-image.sh
+source "$REPO_ROOT/scripts/lib/microvm-image.sh"
+
+echo "$TAG repo root: $REPO_ROOT"
+echo "$TAG build dir: $BUILD_DIR"
+
+microvm_check_tools
+
+for f in "$INIT_SH" "$INITTAB" "$RUN_AGENT"; do
+    if [ ! -f "$f" ]; then
+        echo "ERROR: $f not found — cannot build initramfs" >&2
+        exit 1
+    fi
+done
+if [ ! -x "$LAMBDA_AGENT_BIN" ]; then
+    echo "ERROR: $LAMBDA_AGENT_BIN not found — run 'make bin/lambda-agent'" >&2
+    exit 1
+fi
+if ! file -b "$LAMBDA_AGENT_BIN" | grep -q "statically linked"; then
+    echo "ERROR: $LAMBDA_AGENT_BIN is not statically linked — it cannot run on the musl guest" >&2
+    exit 1
+fi
+
+microvm_init_dirs
+microvm_make_rootfs
+microvm_empty_dev
+
+echo "$TAG installing init, inittab, run-agent and lambda-agent..."
+install -m 0755 "$INIT_SH" "$CHROOT_DIR/init"
+install -m 0644 "$INITTAB" "$CHROOT_DIR/etc/inittab"
+mkdir -p "$CHROOT_DIR/usr/local/lib/lambda-runtime" "$CHROOT_DIR/usr/local/bin"
+install -m 0755 "$RUN_AGENT" "$CHROOT_DIR/usr/local/lib/lambda-runtime/run-agent"
+install -m 0755 "$LAMBDA_AGENT_BIN" "$CHROOT_DIR/usr/local/bin/lambda-agent"
+# Function-code mountpoint and scratch space ship empty.
+mkdir -p "$CHROOT_DIR/var/task" "$CHROOT_DIR/tmp" \
+    "$CHROOT_DIR/proc" "$CHROOT_DIR/sys" "$CHROOT_DIR/dev" "$CHROOT_DIR/etc/ssl/certs"
+# The bootstrap runs unprivileged, so / must be traversable (mktemp -d makes it
+# 0700) and nothing it runs from may be group-writable.
+chmod 0755 "$CHROOT_DIR" "$CHROOT_DIR/var/task" "$CHROOT_DIR/usr/local/lib/lambda-runtime"
+
+microvm_locate_kernel
+microvm_strip_modules
+microvm_strip_rootfs
+microvm_stage_kernel
+microvm_pack_initramfs
+
+# Nothing reaches $BUILD_DIR until the staged pair passes these checks.
+for out in vmlinuz initramfs.cpio.gz; do
+    if [ ! -s "$STAGE_DIR/$out" ]; then
+        echo "ERROR: staged $out is missing or empty" >&2
+        exit 1
+    fi
+done
+if ! gzip -t "$STAGE_DIR/initramfs.cpio.gz"; then
+    echo "ERROR: staged initramfs.cpio.gz is not valid gzip" >&2
+    exit 1
+fi
+mv -f "$STAGE_DIR/vmlinuz" "$BUILD_DIR/vmlinuz"
+mv -f "$STAGE_DIR/initramfs.cpio.gz" "$BUILD_DIR/initramfs.cpio.gz"
+echo ""
+echo "$TAG done."

scripts/build-sqs-store-microvm-image.sh

@@ -18,6 +18,7 @@
 #   1. apk available on host → install directly into a temp chroot (Alpine CI/CD)
 #   2. docker available      → run alpine container, export filesystem
 #   3. podman available      → same via podman
+# shellcheck disable=SC2034 # variables are consumed by the sourced library
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
@@ -44,21 +45,19 @@ ALPINE_PACKAGES="busybox linux-virt iproute2 ca-certificates e2fsprogs kmod"
 # button (QMP system_powerdown) into a signal busybox init handles as poweroff.
 MICROVM_EXTRA_MODULES="${MICROVM_EXTRA_MODULES:-ext4 tiny_power_button}"
 
+# Module file globs kept on top of virtio*/*fw_cfg*: virtio_net deps for live migration.
+MICROVM_EXTRA_KEEP_GLOBS=("*failover*.ko*")
+# Modules the image modprobe must resolve, beyond the extras.
+MICROVM_GATE_MODULES="qemu_fw_cfg virtio_net"
+
+# shellcheck source=lib/microvm-image.sh
+source "$REPO_ROOT/scripts/lib/microvm-image.sh"
+
 echo "$TAG repo root: $REPO_ROOT"
 echo "$TAG build dir: $BUILD_DIR"
 mkdir -p "$BUILD_DIR"
 
-# --- Verify non-negotiable tools ---
-# fakeroot is mandatory: device nodes are written into the cpio archive inside a
-# faked-root environment so no real device node is ever created on the host
-# filesystem (which previously required sudo mknod and risked clobbering host
-# /dev/null on cleanup).
-for tool in cpio gzip find fakeroot; do
-    if ! command -v "$tool" >/dev/null 2>&1; then
-        echo "ERROR: required tool not found: $tool" >&2
-        exit 1
-    fi
-done
+microvm_check_tools
 
 if [ ! -x "$SQS_WORKER_BIN" ]; then
     echo "ERROR: $SQS_WORKER_BIN not found — run 'make bin/sqs-worker'" >&2
@@ -90,100 +89,11 @@ if [ "$nats_server_version" != "$NATS_SERVER_PIN" ]; then
 fi
 echo "$TAG nats-server: $nats_server_version (matches go.mod)"
 
-# --- Create chroot ---
-CHROOT_DIR=$(mktemp -d)
-# Guard: every destructive op below targets "$CHROOT_DIR/...". An empty or
-# unexpected value would aim rm -rf at the host (e.g. /dev). Refuse anything that
-# is not a freshly-created temp dir.
-case "${CHROOT_DIR:?mktemp -d returned empty}" in
-    /tmp/*|/var/tmp/*|"${TMPDIR:-/nonexistent}"/*) ;;
-    *) echo "ERROR: refusing to use unexpected CHROOT_DIR: $CHROOT_DIR" >&2; exit 1 ;;
-esac
-# Outputs are written here first, beside $BUILD_DIR so publishing is a rename.
-STAGE_DIR=$(mktemp -d "$BUILD_DIR/.stage.XXXXXX")
-CONTAINER_TOOL=""
-CONTAINER_CID=""
-cleanup() {
-    echo "$TAG cleaning up chroot: $CHROOT_DIR"
-    rm -rf "$CHROOT_DIR"
-    rm -rf "${STAGE_DIR:?}"
-    if [ -n "$CONTAINER_CID" ] && [ -n "$CONTAINER_TOOL" ]; then
-        echo "$TAG cleaning up container: $CONTAINER_CID"
-        $CONTAINER_TOOL rm -f "$CONTAINER_CID" >/dev/null 2>&1 || true
-    fi
-}
-trap cleanup EXIT
+microvm_init_dirs
 
-# --- Populate rootfs ---
-# Strategy 1: native apk (Alpine hosts, CI)
-if command -v apk >/dev/null 2>&1; then
-    echo "$TAG rootfs: native apk"
-    mkdir -p \
-        "$CHROOT_DIR/etc/apk" \
-        "$CHROOT_DIR/lib/apk/db" \
-        "$CHROOT_DIR/var/cache/apk"
-    cat > "$CHROOT_DIR/etc/apk/repositories" <<EOF
-${ALPINE_MIRROR}/v${ALPINE_VERSION}/main
-${ALPINE_MIRROR}/v${ALPINE_VERSION}/community
-EOF
-    apk add \
-        --root "$CHROOT_DIR" \
-        --arch x86_64 \
-        --no-cache \
-        --allow-untrusted \
-        --initdb \
-        $ALPINE_PACKAGES
+microvm_make_rootfs
 
-# Strategy 2: docker -> BuildKit local export.
-# Do NOT use `docker export` here. On Docker's overlay2 driver it leaks the
-# container's overlay mount (the mount survives `docker rm`), orphaning layers
-# under /var/lib/docker that only a daemon restart reclaims; on a busy CI host
-# this grows without bound. `buildx --output type=local` writes the image
-# filesystem straight to a directory and releases every mount, so nothing leaks.
-elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
-    echo "$TAG rootfs: docker buildx --output type=local (alpine:${ALPINE_VERSION})"
-    BUILD_CTX=$(mktemp -d)
-    cat > "$BUILD_CTX/Dockerfile" <<DOCKERFILE
-FROM alpine:${ALPINE_VERSION}
-RUN apk add --no-cache ${ALPINE_PACKAGES}
-DOCKERFILE
-    docker buildx build \
-        --platform linux/amd64 \
-        --output "type=local,dest=${CHROOT_DIR}" \
-        "$BUILD_CTX"
-    rm -rf "$BUILD_CTX"
-
-# Strategy 3: podman -> run + export. Podman's store does not exhibit the docker
-# overlay2 export-mount leak, and podman has no buildx.
-elif command -v podman >/dev/null 2>&1; then
-    echo "$TAG rootfs: podman export (alpine:${ALPINE_VERSION})"
-    CONTAINER_TOOL=podman
-    CONTAINER_CID=$(podman run -d \
-        "alpine:${ALPINE_VERSION}" \
-        sh -c "apk add --no-cache ${ALPINE_PACKAGES}")
-
-    exit_code=$(podman wait "$CONTAINER_CID")
-    if [ "$exit_code" != "0" ]; then
-        echo "ERROR: package installation failed in container (exit $exit_code)" >&2
-        exit 1
-    fi
-
-    podman export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR"
-
-else
-    echo "ERROR: no rootfs build tool available." >&2
-    echo "       Install one of: apk (Alpine), docker, podman" >&2
-    exit 1
-fi
-
-# --- /dev: empty mountpoint only ---
-# tar extraction without root silently creates 0-byte regular files for device
-# nodes. Drop them and leave an empty /dev mountpoint. The two static nodes the
-# kernel needs (/dev/console to wire init's stdio, /dev/null) are written into
-# the cpio archive under fakeroot at pack time — NOT created on the host. init
-# mounts devtmpfs over /dev as its first action for everything else at runtime.
-rm -rf "${CHROOT_DIR:?}/dev"
-mkdir "$CHROOT_DIR/dev"
+microvm_empty_dev
 
 # --- Place init script ---
 echo "$TAG installing init.sh as /init..."
@@ -227,30 +137,8 @@ mkdir -p \
     "$CHROOT_DIR/dev" \
     "$CHROOT_DIR/etc/ssl/certs"
 
-# --- Locate kernel ---
-KERNEL_IMG=$(find "$CHROOT_DIR/boot" -name "vmlinuz*" | sort | tail -1)
-if [ -z "$KERNEL_IMG" ]; then
-    echo "ERROR: no vmlinuz found in $CHROOT_DIR/boot" >&2
-    exit 1
-fi
-echo "$TAG kernel image: $KERNEL_IMG"
 
-KERNEL_VER=$(basename "$KERNEL_IMG" | sed 's/vmlinuz-//')
-KERNEL_CONFIG="$CHROOT_DIR/boot/config-${KERNEL_VER}"
-
-# --- Assert fw_cfg module ---
-if ! find "$CHROOT_DIR/lib/modules" \
-        -name "qemu_fw_cfg.ko*" \
-        -o -name "fw_cfg_sysfs.ko*" \
-    | grep -q .; then
-    if ! grep -q "CONFIG_FW_CFG_SYSFS=y" "$KERNEL_CONFIG" 2>/dev/null; then
-        echo "ERROR: qemu_fw_cfg module missing from initramfs — fw_cfg reads will fail at boot" >&2
-        exit 1
-    fi
-    echo "$TAG fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
-else
-    echo "$TAG fw_cfg: module found in lib/modules"
-fi
+microvm_locate_kernel
 
 # --- Assert ext4 userspace tools (data volume) ---
 for fs_tool in mkfs.ext4 e2fsck; do
@@ -261,227 +149,10 @@ for fs_tool in mkfs.ext4 e2fsck; do
 done
 echo "$TAG e2fsprogs: mkfs.ext4 + e2fsck present"
 
-# --- Strip kernel modules (Phase 2 keep-list approach) ---
-# Only modules needed by a virtio-mmio microvm guest are kept:
-#   virtio*.ko*    — virtio bus, net, blk, rng, console, and helpers
-#   *fw_cfg*.ko*   — QEMU fw_cfg sysfs driver (delivers boot blobs)
-#   *failover*.ko* — net_failover + failover (virtio_net deps for live migration)
-#   $MICROVM_EXTRA_MODULES — ext4 by default, plus dependency closure
-# All other loadable modules are removed. Drivers compiled into the kernel
-# (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
-
-# module_path prints a module's kernel/ path from the stock modules.dep, by
-# name or modules.alias; it fails when the module is not loadable.
-module_path() {
-    local kdir="$1" name="${2//-/_}" path alias_target
-    path=$(awk -F: -v n="$name" '{ f = $1; sub(/.*\//, "", f); sub(/\.ko(\.gz|\.xz|\.zst)?$/, "", f);
-        gsub(/-/, "_", f); if (f == n) { print $1; exit } }' "$kdir/modules.dep")
-    if [ -z "$path" ] && [ -f "$kdir/modules.alias" ]; then
-        alias_target=$(awk -v n="$2" '$1 == "alias" && $2 == n { print $3; exit }' "$kdir/modules.alias")
-        [ -n "$alias_target" ] && [ "$alias_target" != "$2" ] && path=$(module_path "$kdir" "$alias_target")
-    fi
-    [ -n "$path" ] && printf '%s\n' "$path"
-}
-
-# module_builtin reports whether a module name is compiled into the kernel.
-module_builtin() {
-    local kdir="$1" name="${2//-/_}"
-    [ -f "$kdir/modules.builtin" ] || return 1
-    awk -v n="$name" '{ f = $0; sub(/.*\//, "", f); sub(/\.ko$/, "", f); gsub(/-/, "_", f);
-        if (f == n) { found = 1; exit } } END { exit !found }' "$kdir/modules.builtin"
-}
-
-# image_kmod runs the image's own kmod (depmod, modprobe) through its musl
-# loader, so module metadata matches the guest's modprobe without a host kmod.
-image_kmod() {
-    "$CHROOT_DIR/lib/ld-musl-x86_64.so.1" --library-path "$CHROOT_DIR/lib:$CHROOT_DIR/usr/lib" \
-        --argv0 "$1" "$CHROOT_DIR/bin/kmod" "${@:2}"
-}
-
-echo "$TAG stripping kernel modules to virtio+fw_cfg+${MICROVM_EXTRA_MODULES} only..."
-for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
-    kver="$(basename "$kver_dir")"
-    kernel_dir="$kver_dir/kernel"
-    [ -d "$kernel_dir" ] || continue
-
-    if [ ! -s "$kver_dir/modules.dep" ]; then
-        echo "ERROR: stock ${kver_dir}modules.dep missing — cannot resolve module dependencies" >&2
-        exit 1
-    fi
-
-    # Extra modules start the closure; the glob families are kept as found.
-    keep_list=$(mktemp)
-    : > "$keep_list"
-    for name in $MICROVM_EXTRA_MODULES; do
-        if ! path=$(module_path "$kver_dir" "$name"); then
-            if module_builtin "$kver_dir" "$name"; then continue; fi
-            echo "ERROR: module $name is neither loadable nor built into $kver" >&2
-            exit 1
-        fi
-        echo "$path" >> "$keep_list"
-    done
-
-    # Close over modules.dep hard dependencies and modules.softdep pre:/post:
-    # names until the set stops growing. A softdep must be loadable or built in.
-    while :; do
-        before=$(sort -u "$keep_list" | wc -l)
-        for path in $(sort -u "$keep_list"); do
-            awk -F': ' -v p="$path" '$1 == p { n = split($2, d, " "); for (i = 1; i <= n; i++) print d[i] }' \
-                "$kver_dir/modules.dep" >> "$keep_list"
-            mod=$(basename "$path"); mod="${mod%%.ko*}"; mod="${mod//-/_}"
-            [ -f "$kver_dir/modules.softdep" ] || continue
-            for soft in $(awk -v n="$mod" '{ m = $2; gsub(/-/, "_", m) }
-                    $1 == "softdep" && m == n { for (i = 3; i <= NF; i++) if ($i != "pre:" && $i != "post:") print $i }' \
-                    "$kver_dir/modules.softdep"); do
-                if soft_path=$(module_path "$kver_dir" "$soft"); then
-                    echo "$soft_path" >> "$keep_list"
-                elif ! module_builtin "$kver_dir" "$soft"; then
-                    echo "ERROR: softdep $soft of $mod is neither loadable nor built into $kver" >&2
-                    exit 1
-                fi
-            done
-        done
-        after=$(sort -u "$keep_list" | wc -l)
-        [ "$before" -eq "$after" ] && break
-    done
-
-    extra_names=$(sort -u "$keep_list" | while read -r p; do m=$(basename "$p"); m="${m%%.ko*}"; echo "${m//-/_}"; done)
-    (cd "$kver_dir" && find kernel -name "virtio*.ko*" -o -name "*fw_cfg*.ko*" \
-        -o -name "*failover*.ko*") >> "$keep_list"
-
-    # Copy the keeper set out, preserving each kernel/ relative path.
-    tmp_keep=$(mktemp -d)
-    while read -r rel; do
-        rel="${rel#kernel/}"
-        mkdir -p "$tmp_keep/$(dirname "$rel")"
-        cp "$kernel_dir/$rel" "$tmp_keep/$rel"
-    done < <(sort -u "$keep_list")
-    rm -f "$keep_list"
-
-    # Replace kernel module tree with the keeper set.
-    rm -rf "$kernel_dir"
-    mkdir -p "$kernel_dir"
-    # Restore each kept file into the same relative path.
-    (cd "$tmp_keep" && find . -name "*.ko*" | while read -r f; do
-        dest_dir="$kernel_dir/$(dirname "${f#./}")"
-        mkdir -p "$dest_dir"
-        cp "$f" "$dest_dir/"
-    done)
-    rm -rf "$tmp_keep"
-
-    # Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
-    # kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
-    # produces an EMPTY modules.dep from compressed modules — the guest's
-    # modprobe then no-ops and qemu_fw_cfg never loads. Plain .ko sidesteps the
-    # host-kmod dependency entirely and the guest needs no decompressor either.
-    find "$kernel_dir" -name "*.ko.gz" -exec gunzip -f {} +
-
-    # Independent closure gate: each extra-closure module's own .modinfo depends=
-    # must name a kept module or a built-in, so a resolver bug fails here.
-    kept_names=$(find "$kernel_dir" -name "*.ko" -exec basename {} .ko \; | tr '-' '_' | sort -u)
-    while read -r ko; do
-        ko_name=$(basename "$ko" .ko); ko_name="${ko_name//-/_}"
-        grep -qx "$ko_name" <<< "$extra_names" || continue
-        deps=$(tr '\0' '\n' < "$ko" | grep -a -m1 '^depends=' | cut -d= -f2 | tr ',' ' ' || true)
-        for dep in $deps; do
-            dep="${dep//-/_}"
-            if ! grep -qx "$dep" <<< "$kept_names" && ! module_builtin "$kver_dir" "$dep"; then
-                echo "ERROR: $(basename "$ko") depends on $dep, which is neither kept nor built in" >&2
-                exit 1
-            fi
-        done
-    done < <(find "$kernel_dir" -name "*.ko")
-    for name in $MICROVM_EXTRA_MODULES; do
-        if ! grep -qx "${name//-/_}" <<< "$kept_names" && ! module_builtin "$kver_dir" "$name"; then
-            echo "ERROR: required module $name missing from the stripped module tree" >&2
-            exit 1
-        fi
-    done
-    echo "$TAG extra module closure: $(echo "$extra_names" | tr '\n' ' ')"
-    echo "$TAG modules kept: $(echo "$kept_names" | tr '\n' ' ')"
-
-    # Rebuild module metadata with the image's own kmod, the binary the guest's
-    # modprobe runs, so no host depmod is needed. Stock indexes still name .ko.gz
-    # files, so they are removed first and a failed rebuild cannot ship them.
-    for f in dep dep.bin alias alias.bin softdep weakdep symbols symbols.bin \
-        builtin.bin builtin.alias.bin devname; do
-        rm -f "$kver_dir/modules.$f"
-    done
-    if ! image_kmod depmod -b "$CHROOT_DIR" "$kver"; then
-        echo "ERROR: image depmod failed for $kver" >&2
-        exit 1
-    fi
-    if [ ! -s "$kver_dir/modules.dep" ] || [ ! -s "$kver_dir/modules.dep.bin" ]; then
-        echo "ERROR: ${kver_dir}modules.dep or modules.dep.bin missing or empty after depmod" >&2
-        exit 1
-    fi
-    # Resolution gate: the image's modprobe must resolve every required module
-    # to plain .ko files present in the stripped tree.
-    for name in $MICROVM_EXTRA_MODULES qemu_fw_cfg virtio_net; do
-        module_builtin "$kver_dir" "$name" && continue
-        if ! plan=$(image_kmod modprobe -d "$CHROOT_DIR" -S "$kver" --show-depends "$name"); then
-            echo "ERROR: image modprobe cannot resolve $name for $kver" >&2
-            exit 1
-        fi
-        while read -r verb path _; do
-            [ "$verb" = "insmod" ] || continue
-            case "$path" in
-                "$CHROOT_DIR/lib/modules/$kver/kernel/"*.ko) [ -f "$path" ] && continue ;;
-            esac
-            echo "ERROR: modprobe plan for $name names $path, not a kept plain .ko" >&2
-            exit 1
-        done <<< "$plan"
-        echo "$TAG modprobe $name: $(awk '{ n = split($2, a, "/"); printf "%s ", a[n] }' <<< "$plan")"
-    done
-done
-
-# --- Strip package-manager artifacts (not needed at runtime) ---
-echo "$TAG stripping package manager artifacts..."
-rm -rf \
-    "$CHROOT_DIR/lib/apk" \
-    "$CHROOT_DIR/var/cache/apk" \
-    "$CHROOT_DIR/etc/apk" \
-    "$CHROOT_DIR/usr/share/man" \
-    "$CHROOT_DIR/usr/share/doc" \
-    "$CHROOT_DIR/usr/share/apk" \
-    "$CHROOT_DIR/usr/include" \
-    "$CHROOT_DIR/usr/lib/pkgconfig"
-
-# --- Strip debug symbols from binaries ---
-# nats-server in /usr/local/bin is left as built (-s -w), so the shipped bytes
-# are the ones whose version was checked above.
-echo "$TAG stripping debug symbols from binaries..."
-find "$CHROOT_DIR/usr/sbin" "$CHROOT_DIR/usr/bin" \
-     "$CHROOT_DIR/sbin" "$CHROOT_DIR/bin" \
-    -type f 2>/dev/null | while read -r bin; do
-    strip --strip-all "$bin" 2>/dev/null || true
-done
-
-# --- Copy vmlinuz out before stripping /boot from chroot ---
-VMLINUZ_OUT="$STAGE_DIR/vmlinuz"
-echo "$TAG copying kernel: $VMLINUZ_OUT"
-cp "$KERNEL_IMG" "$VMLINUZ_OUT"
-
-# Drop /boot from the chroot — vmlinuz, Alpine's stock initramfs-virt, System.map
-# and kernel config all live here (~25 MiB). None are needed at runtime: the
-# guest already has the kernel loaded by QEMU's -kernel, and Alpine's initramfs
-# is unused because we replace it with our own /init. Leaving /boot in the cpio
-# nearly doubles uncompressed initramfs size and forces a 512 MiB guest just to
-# survive decompression.
-rm -rf "${CHROOT_DIR:?}/boot"
-
-# --- Build initramfs ---
-INITRAMFS_OUT="$STAGE_DIR/initramfs.cpio.gz"
-echo "$TAG building initramfs: $INITRAMFS_OUT"
-# Create the static device nodes and pack the archive inside a single fakeroot
-# session: mknod/cpio see faked device entries and write real char-device
-# records into the cpio, while the host filesystem never gets an actual node.
-fakeroot sh -c '
-    cd "$1" || exit 1
-    mknod -m 600 dev/console c 5 1
-    mknod -m 666 dev/null    c 1 3
-    find . | cpio --quiet -o -H newc | gzip -9 > "$2"
-' _ "$CHROOT_DIR" "$INITRAMFS_OUT"
+microvm_strip_modules
+microvm_strip_rootfs
+microvm_stage_kernel
+microvm_pack_initramfs
 
 # --- Gate the staged pair, then publish it ---
 # Nothing reaches $BUILD_DIR until every gate passes, so a failed build cannot

scripts/lib/microvm-image.sh

@@ -0,0 +1,391 @@
+#!/bin/bash
+# microvm-image.sh — shared steps for the microVM image builders, sourced by
+# build-sqs-store-microvm-image.sh and build-lambda-runtime-microvm-image.sh.
+#
+# The caller sets before use:
+#   TAG                      log prefix
+#   BUILD_DIR                output directory; STAGE_DIR is created beside it
+#   ALPINE_VERSION, ALPINE_MIRROR, ALPINE_PACKAGES
+#   MICROVM_EXTRA_MODULES    modules kept by name, with their dependency closure
+#   MICROVM_GATE_MODULES     modules the image modprobe must resolve, beyond the extras
+#   MICROVM_EXTRA_KEEP_GLOBS bash array of module file globs kept on top of virtio*/*fw_cfg*
+# Functions set CHROOT_DIR, STAGE_DIR, KERNEL_IMG and KERNEL_VER for the caller.
+
+
+microvm_check_tools() {
+    # --- Verify non-negotiable tools ---
+    # fakeroot is mandatory: device nodes are written into the cpio archive inside a
+    # faked-root environment so no real device node is ever created on the host
+    # filesystem (which previously required sudo mknod and risked clobbering host
+    # /dev/null on cleanup).
+    for tool in cpio gzip find fakeroot; do
+        if ! command -v "$tool" >/dev/null 2>&1; then
+            echo "ERROR: required tool not found: $tool" >&2
+            exit 1
+        fi
+    done
+}
+
+microvm_init_dirs() {
+    # --- Create chroot ---
+    CHROOT_DIR=$(mktemp -d)
+    # Guard: every destructive op below targets "$CHROOT_DIR/...". An empty or
+    # unexpected value would aim rm -rf at the host (e.g. /dev). Refuse anything that
+    # is not a freshly-created temp dir.
+    case "${CHROOT_DIR:?mktemp -d returned empty}" in
+        /tmp/*|/var/tmp/*|"${TMPDIR:-/nonexistent}"/*) ;;
+        *) echo "ERROR: refusing to use unexpected CHROOT_DIR: $CHROOT_DIR" >&2; exit 1 ;;
+    esac
+    # Outputs are written here first, beside $BUILD_DIR so publishing is a rename.
+    STAGE_DIR=$(mktemp -d "$BUILD_DIR/.stage.XXXXXX")
+    CONTAINER_TOOL=""
+    CONTAINER_CID=""
+    trap microvm_cleanup EXIT
+}
+
+microvm_cleanup() {
+        echo "$TAG cleaning up chroot: $CHROOT_DIR"
+        rm -rf "$CHROOT_DIR"
+        rm -rf "${STAGE_DIR:?}"
+        if [ -n "$CONTAINER_CID" ] && [ -n "$CONTAINER_TOOL" ]; then
+            echo "$TAG cleaning up container: $CONTAINER_CID"
+            $CONTAINER_TOOL rm -f "$CONTAINER_CID" >/dev/null 2>&1 || true
+        fi
+}
+
+# microvm_make_rootfs populates $CHROOT_DIR from ALPINE_PACKAGES.
+microvm_make_rootfs() {
+    # --- Populate rootfs ---
+    # Strategy 1: native apk (Alpine hosts, CI)
+    if command -v apk >/dev/null 2>&1; then
+        echo "$TAG rootfs: native apk"
+        mkdir -p \
+            "$CHROOT_DIR/etc/apk" \
+            "$CHROOT_DIR/lib/apk/db" \
+            "$CHROOT_DIR/var/cache/apk"
+        cat > "$CHROOT_DIR/etc/apk/repositories" <<EOF
+${ALPINE_MIRROR}/v${ALPINE_VERSION}/main
+${ALPINE_MIRROR}/v${ALPINE_VERSION}/community
+EOF
+        apk add \
+            --root "$CHROOT_DIR" \
+            --arch x86_64 \
+            --no-cache \
+            --allow-untrusted \
+            --initdb \
+            $ALPINE_PACKAGES
+
+    # Strategy 2: docker -> BuildKit local export.
+    # Do NOT use `docker export` here. On Docker's overlay2 driver it leaks the
+    # container's overlay mount (the mount survives `docker rm`), orphaning layers
+    # under /var/lib/docker that only a daemon restart reclaims; on a busy CI host
+    # this grows without bound. `buildx --output type=local` writes the image
+    # filesystem straight to a directory and releases every mount, so nothing leaks.
+    elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
+        echo "$TAG rootfs: docker buildx --output type=local (alpine:${ALPINE_VERSION})"
+        BUILD_CTX=$(mktemp -d)
+        cat > "$BUILD_CTX/Dockerfile" <<DOCKERFILE
+FROM alpine:${ALPINE_VERSION}
+RUN apk add --no-cache ${ALPINE_PACKAGES}
+DOCKERFILE
+        docker buildx build \
+            --platform linux/amd64 \
+            --output "type=local,dest=${CHROOT_DIR}" \
+            "$BUILD_CTX"
+        rm -rf "$BUILD_CTX"
+
+    # Strategy 3: podman -> run + export. Podman's store does not exhibit the docker
+    # overlay2 export-mount leak, and podman has no buildx.
+    elif command -v podman >/dev/null 2>&1; then
+        echo "$TAG rootfs: podman export (alpine:${ALPINE_VERSION})"
+        CONTAINER_TOOL=podman
+        CONTAINER_CID=$(podman run -d \
+            "alpine:${ALPINE_VERSION}" \
+            sh -c "apk add --no-cache ${ALPINE_PACKAGES}")
+
+        exit_code=$(podman wait "$CONTAINER_CID")
+        if [ "$exit_code" != "0" ]; then
+            echo "ERROR: package installation failed in container (exit $exit_code)" >&2
+            exit 1
+        fi
+
+        podman export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR"
+
+    else
+        echo "ERROR: no rootfs build tool available." >&2
+        echo "       Install one of: apk (Alpine), docker, podman" >&2
+        exit 1
+    fi
+}
+
+microvm_empty_dev() {
+    # --- /dev: empty mountpoint only ---
+    # tar extraction without root silently creates 0-byte regular files for device
+    # nodes. Drop them and leave an empty /dev mountpoint. The two static nodes the
+    # kernel needs (/dev/console to wire init's stdio, /dev/null) are written into
+    # the cpio archive under fakeroot at pack time — NOT created on the host. init
+    # mounts devtmpfs over /dev as its first action for everything else at runtime.
+    rm -rf "${CHROOT_DIR:?}/dev"
+    mkdir "$CHROOT_DIR/dev"
+}
+
+# microvm_locate_kernel finds the kernel and asserts fw_cfg is loadable or built in.
+microvm_locate_kernel() {
+    # --- Locate kernel ---
+    KERNEL_IMG=$(find "$CHROOT_DIR/boot" -name "vmlinuz*" | sort | tail -1)
+    if [ -z "$KERNEL_IMG" ]; then
+        echo "ERROR: no vmlinuz found in $CHROOT_DIR/boot" >&2
+        exit 1
+    fi
+    echo "$TAG kernel image: $KERNEL_IMG"
+
+    KERNEL_VER=$(basename "$KERNEL_IMG" | sed 's/vmlinuz-//')
+    KERNEL_CONFIG="$CHROOT_DIR/boot/config-${KERNEL_VER}"
+
+    # --- Assert fw_cfg module ---
+    if ! find "$CHROOT_DIR/lib/modules" \
+            -name "qemu_fw_cfg.ko*" \
+            -o -name "fw_cfg_sysfs.ko*" \
+        | grep -q .; then
+        if ! grep -q "CONFIG_FW_CFG_SYSFS=y" "$KERNEL_CONFIG" 2>/dev/null; then
+            echo "ERROR: qemu_fw_cfg module missing from initramfs — fw_cfg reads will fail at boot" >&2
+            exit 1
+        fi
+        echo "$TAG fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
+    else
+        echo "$TAG fw_cfg: module found in lib/modules"
+    fi
+}
+
+# --- Strip kernel modules (Phase 2 keep-list approach) ---
+# Only modules needed by a virtio-mmio microvm guest are kept:
+#   virtio*.ko*    — virtio bus, net, blk, rng, console, and helpers
+#   *fw_cfg*.ko*   — QEMU fw_cfg sysfs driver (delivers boot blobs)
+#   $MICROVM_EXTRA_KEEP_GLOBS — caller-named module file globs
+#   $MICROVM_EXTRA_MODULES — caller-named modules, plus dependency closure
+# All other loadable modules are removed. Drivers compiled into the kernel
+# (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
+
+# module_path prints a module's kernel/ path from the stock modules.dep, by
+# name or modules.alias; it fails when the module is not loadable.
+module_path() {
+    local kdir="$1" name="${2//-/_}" path alias_target
+    path=$(awk -F: -v n="$name" '{ f = $1; sub(/.*\//, "", f); sub(/\.ko(\.gz|\.xz|\.zst)?$/, "", f);
+        gsub(/-/, "_", f); if (f == n) { print $1; exit } }' "$kdir/modules.dep")
+    if [ -z "$path" ] && [ -f "$kdir/modules.alias" ]; then
+        alias_target=$(awk -v n="$2" '$1 == "alias" && $2 == n { print $3; exit }' "$kdir/modules.alias")
+        [ -n "$alias_target" ] && [ "$alias_target" != "$2" ] && path=$(module_path "$kdir" "$alias_target")
+    fi
+    [ -n "$path" ] && printf '%s\n' "$path"
+}
+
+# module_builtin reports whether a module name is compiled into the kernel.
+module_builtin() {
+    local kdir="$1" name="${2//-/_}"
+    [ -f "$kdir/modules.builtin" ] || return 1
+    awk -v n="$name" '{ f = $0; sub(/.*\//, "", f); sub(/\.ko$/, "", f); gsub(/-/, "_", f);
+        if (f == n) { found = 1; exit } } END { exit !found }' "$kdir/modules.builtin"
+}
+
+# image_kmod runs the image's own kmod (depmod, modprobe) through its musl
+# loader, so module metadata matches the guest's modprobe without a host kmod.
+image_kmod() {
+    "$CHROOT_DIR/lib/ld-musl-x86_64.so.1" --library-path "$CHROOT_DIR/lib:$CHROOT_DIR/usr/lib" \
+        --argv0 "$1" "$CHROOT_DIR/bin/kmod" "${@:2}"
+}
+
+microvm_strip_modules() {
+    echo "$TAG stripping kernel modules to virtio+fw_cfg+${MICROVM_EXTRA_MODULES} only..."
+    for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
+        kver="$(basename "$kver_dir")"
+        kernel_dir="$kver_dir/kernel"
+        [ -d "$kernel_dir" ] || continue
+
+        if [ ! -s "$kver_dir/modules.dep" ]; then
+            echo "ERROR: stock ${kver_dir}modules.dep missing — cannot resolve module dependencies" >&2
+            exit 1
+        fi
+
+        # Extra modules start the closure; the glob families are kept as found.
+        keep_list=$(mktemp)
+        : > "$keep_list"
+        for name in $MICROVM_EXTRA_MODULES; do
+            if ! path=$(module_path "$kver_dir" "$name"); then
+                if module_builtin "$kver_dir" "$name"; then continue; fi
+                echo "ERROR: module $name is neither loadable nor built into $kver" >&2
+                exit 1
+            fi
+            echo "$path" >> "$keep_list"
+        done
+
+        # Close over modules.dep hard dependencies and modules.softdep pre:/post:
+        # names until the set stops growing. A softdep must be loadable or built in.
+        while :; do
+            before=$(sort -u "$keep_list" | wc -l)
+            for path in $(sort -u "$keep_list"); do
+                awk -F': ' -v p="$path" '$1 == p { n = split($2, d, " "); for (i = 1; i <= n; i++) print d[i] }' \
+                    "$kver_dir/modules.dep" >> "$keep_list"
+                mod=$(basename "$path"); mod="${mod%%.ko*}"; mod="${mod//-/_}"
+                [ -f "$kver_dir/modules.softdep" ] || continue
+                for soft in $(awk -v n="$mod" '{ m = $2; gsub(/-/, "_", m) }
+                        $1 == "softdep" && m == n { for (i = 3; i <= NF; i++) if ($i != "pre:" && $i != "post:") print $i }' \
+                        "$kver_dir/modules.softdep"); do
+                    if soft_path=$(module_path "$kver_dir" "$soft"); then
+                        echo "$soft_path" >> "$keep_list"
+                    elif ! module_builtin "$kver_dir" "$soft"; then
+                        echo "ERROR: softdep $soft of $mod is neither loadable nor built into $kver" >&2
+                        exit 1
+                    fi
+                done
+            done
+            after=$(sort -u "$keep_list" | wc -l)
+            [ "$before" -eq "$after" ] && break
+        done
+
+        extra_names=$(sort -u "$keep_list" | while read -r p; do m=$(basename "$p"); m="${m%%.ko*}"; echo "${m//-/_}"; done)
+        find_args=(-name "virtio*.ko*" -o -name "*fw_cfg*.ko*")
+        for glob in ${MICROVM_EXTRA_KEEP_GLOBS[@]+"${MICROVM_EXTRA_KEEP_GLOBS[@]}"}; do
+            find_args+=(-o -name "$glob")
+        done
+        (cd "$kver_dir" && find kernel "${find_args[@]}") >> "$keep_list"
+
+        # Copy the keeper set out, preserving each kernel/ relative path.
+        tmp_keep=$(mktemp -d)
+        while read -r rel; do
+            rel="${rel#kernel/}"
+            mkdir -p "$tmp_keep/$(dirname "$rel")"
+            cp "$kernel_dir/$rel" "$tmp_keep/$rel"
+        done < <(sort -u "$keep_list")
+        rm -f "$keep_list"
+
+        # Replace kernel module tree with the keeper set.
+        rm -rf "$kernel_dir"
+        mkdir -p "$kernel_dir"
+        # Restore each kept file into the same relative path.
+        (cd "$tmp_keep" && find . -name "*.ko*" | while read -r f; do
+            dest_dir="$kernel_dir/$(dirname "${f#./}")"
+            mkdir -p "$dest_dir"
+            cp "$f" "$dest_dir/"
+        done)
+        rm -rf "$tmp_keep"
+
+        # Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
+        # kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
+        # produces an EMPTY modules.dep from compressed modules — the guest's
+        # modprobe then no-ops and qemu_fw_cfg never loads. Plain .ko sidesteps the
+        # host-kmod dependency entirely and the guest needs no decompressor either.
+        find "$kernel_dir" -name "*.ko.gz" -exec gunzip -f {} +
+
+        # Independent closure gate: each extra-closure module's own .modinfo depends=
+        # must name a kept module or a built-in, so a resolver bug fails here.
+        kept_names=$(find "$kernel_dir" -name "*.ko" -exec basename {} .ko \; | tr '-' '_' | sort -u)
+        while read -r ko; do
+            ko_name=$(basename "$ko" .ko); ko_name="${ko_name//-/_}"
+            grep -qx "$ko_name" <<< "$extra_names" || continue
+            deps=$(tr '\0' '\n' < "$ko" | grep -a -m1 '^depends=' | cut -d= -f2 | tr ',' ' ' || true)
+            for dep in $deps; do
+                dep="${dep//-/_}"
+                if ! grep -qx "$dep" <<< "$kept_names" && ! module_builtin "$kver_dir" "$dep"; then
+                    echo "ERROR: $(basename "$ko") depends on $dep, which is neither kept nor built in" >&2
+                    exit 1
+                fi
+            done
+        done < <(find "$kernel_dir" -name "*.ko")
+        for name in $MICROVM_EXTRA_MODULES; do
+            if ! grep -qx "${name//-/_}" <<< "$kept_names" && ! module_builtin "$kver_dir" "$name"; then
+                echo "ERROR: required module $name missing from the stripped module tree" >&2
+                exit 1
+            fi
+        done
+        echo "$TAG extra module closure: $(echo "$extra_names" | tr '\n' ' ')"
+        echo "$TAG modules kept: $(echo "$kept_names" | tr '\n' ' ')"
+
+        # Rebuild module metadata with the image's own kmod, the binary the guest's
+        # modprobe runs, so no host depmod is needed. Stock indexes still name .ko.gz
+        # files, so they are removed first and a failed rebuild cannot ship them.
+        for f in dep dep.bin alias alias.bin softdep weakdep symbols symbols.bin \
+            builtin.bin builtin.alias.bin devname; do
+            rm -f "$kver_dir/modules.$f"
+        done
+        if ! image_kmod depmod -b "$CHROOT_DIR" "$kver"; then
+            echo "ERROR: image depmod failed for $kver" >&2
+            exit 1
+        fi
+        if [ ! -s "$kver_dir/modules.dep" ] || [ ! -s "$kver_dir/modules.dep.bin" ]; then
+            echo "ERROR: ${kver_dir}modules.dep or modules.dep.bin missing or empty after depmod" >&2
+            exit 1
+        fi
+        # Resolution gate: the image's modprobe must resolve every required module
+        # to plain .ko files present in the stripped tree.
+        for name in $MICROVM_EXTRA_MODULES $MICROVM_GATE_MODULES; do
+            module_builtin "$kver_dir" "$name" && continue
+            if ! plan=$(image_kmod modprobe -d "$CHROOT_DIR" -S "$kver" --show-depends "$name"); then
+                echo "ERROR: image modprobe cannot resolve $name for $kver" >&2
+                exit 1
+            fi
+            while read -r verb path _; do
+                [ "$verb" = "insmod" ] || continue
+                case "$path" in
+                    "$CHROOT_DIR/lib/modules/$kver/kernel/"*.ko) [ -f "$path" ] && continue ;;
+                esac
+                echo "ERROR: modprobe plan for $name names $path, not a kept plain .ko" >&2
+                exit 1
+            done <<< "$plan"
+            echo "$TAG modprobe $name: $(awk '{ n = split($2, a, "/"); printf "%s ", a[n] }' <<< "$plan")"
+        done
+    done
+}
+
+# microvm_strip_rootfs drops package-manager artifacts and debug symbols.
+microvm_strip_rootfs() {
+    # --- Strip package-manager artifacts (not needed at runtime) ---
+    echo "$TAG stripping package manager artifacts..."
+    rm -rf \
+        "$CHROOT_DIR/lib/apk" \
+        "$CHROOT_DIR/var/cache/apk" \
+        "$CHROOT_DIR/etc/apk" \
+        "$CHROOT_DIR/usr/share/man" \
+        "$CHROOT_DIR/usr/share/doc" \
+        "$CHROOT_DIR/usr/share/apk" \
+        "$CHROOT_DIR/usr/include" \
+        "$CHROOT_DIR/usr/lib/pkgconfig"
+
+    # --- Strip debug symbols from binaries ---
+    echo "$TAG stripping debug symbols from binaries..."
+    find "$CHROOT_DIR/usr/sbin" "$CHROOT_DIR/usr/bin" \
+         "$CHROOT_DIR/sbin" "$CHROOT_DIR/bin" \
+        -type f 2>/dev/null | while read -r bin; do
+        strip --strip-all "$bin" 2>/dev/null || true
+    done
+}
+
+# microvm_stage_kernel copies vmlinuz to $STAGE_DIR, then drops /boot from the chroot.
+microvm_stage_kernel() {
+    # --- Copy vmlinuz out before stripping /boot from chroot ---
+    VMLINUZ_OUT="$STAGE_DIR/vmlinuz"
+    echo "$TAG copying kernel: $VMLINUZ_OUT"
+    cp "$KERNEL_IMG" "$VMLINUZ_OUT"
+
+    # Drop /boot from the chroot — vmlinuz, Alpine's stock initramfs-virt, System.map
+    # and kernel config all live here (~25 MiB). None are needed at runtime: the
+    # guest already has the kernel loaded by QEMU's -kernel, and Alpine's initramfs
+    # is unused because we replace it with our own /init. Leaving /boot in the cpio
+    # nearly doubles uncompressed initramfs size and forces a 512 MiB guest just to
+    # survive decompression.
+    rm -rf "${CHROOT_DIR:?}/boot"
+}
+
+microvm_pack_initramfs() {
+    # --- Build initramfs ---
+    INITRAMFS_OUT="$STAGE_DIR/initramfs.cpio.gz"
+    echo "$TAG building initramfs: $INITRAMFS_OUT"
+    # Create the static device nodes and pack the archive inside a single fakeroot
+    # session: mknod/cpio see faked device entries and write real char-device
+    # records into the cpio, while the host filesystem never gets an actual node.
+    fakeroot sh -c '
+        cd "$1" || exit 1
+        mknod -m 600 dev/console c 5 1
+        mknod -m 666 dev/null    c 1 3
+        find . | cpio --quiet -o -H newc | gzip -9 > "$2"
+    ' _ "$CHROOT_DIR" "$INITRAMFS_OUT"
+}

spinifex/agents/ecs/agent/credendpoint.go

@@ -47,6 +47,13 @@ type credResponse struct {
 
 // credEntry caches one task's assumed-role credentials alongside the role it was
 // assumed from.
+// credRole is the task role a credID serves and the task it belongs to; the
+// task ID names the STS session, since the credID is a secret.
+type credRole struct {
+	roleARN string
+	taskID  string
+}
+
 type credEntry struct {
 	roleARN string
 	creds   stsauth.Credentials
@@ -71,7 +78,7 @@ type credEndpoint struct {
 	assume func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error)
 
 	mu         sync.Mutex
-	roles      map[string]string    // credID -> taskRoleARN
+	roles      map[string]credRole  // credID -> task role
 	cache      map[string]credEntry // credID -> cached creds
 	httpClient *http.Client
 
@@ -98,7 +105,7 @@ func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayUR
 		port:       port,
 		proxyPort:  defaultCredProxyPort,
 		run:        run,
-		roles:      map[string]string{},
+		roles:      map[string]credRole{},
 		cache:      map[string]credEntry{},
 	}
 	c.assume = c.assumeOverGateway
@@ -107,12 +114,12 @@ func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayUR
 
 // Register associates a credID with the task role it serves; Deregister removes
 // it and drops any cached credentials. Both are no-ops for an empty credID.
-func (c *credEndpoint) Register(credID, roleARN string) {
+func (c *credEndpoint) Register(credID, taskID, roleARN string) {
 	if credID == "" || roleARN == "" {
 		return
 	}
 	c.mu.Lock()
-	c.roles[credID] = roleARN
+	c.roles[credID] = credRole{roleARN: roleARN, taskID: taskID}
 	c.mu.Unlock()
 }
 
@@ -142,25 +149,25 @@ func (c *credEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "not found", http.StatusNotFound)
 		return
 	}
-	creds, roleARN, err := c.fetch(r.Context(), credID)
+	creds, role, err := c.fetch(r.Context(), credID)
 	if err != nil {
 		if errors.Is(err, errUnknownCredID) {
 			http.Error(w, "not found", http.StatusNotFound)
 			return
 		}
-		slog.Error("ecs-agent: assume task role failed", "credId", credID, "err", err)
+		slog.Error("ecs-agent: assume task role failed", "task", role.taskID, "role", role.roleARN, "err", err)
 		http.Error(w, "credentials unavailable", http.StatusBadGateway)
 		return
 	}
 	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(credResponse{
-		RoleArn:         roleARN,
+		RoleArn:         role.roleARN,
 		AccessKeyID:     creds.AccessKeyID,
 		SecretAccessKey: creds.SecretAccessKey,
 		Token:           creds.SessionToken,
 		Expiration:      creds.Expiration.UTC().Format(time.RFC3339),
 	}); err != nil {
-		slog.Warn("ecs-agent: encode credential response", "credId", credID, "err", err)
+		slog.Warn("ecs-agent: encode credential response", "task", role.taskID, "err", err)
 	}
 }
 
@@ -169,28 +176,28 @@ var errUnknownCredID = fmt.Errorf("unknown credId")
 
 // fetch returns valid credentials for credID, reusing the cache until within the
 // refresh margin and otherwise assuming the task role over the gateway.
-func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, string, error) {
+func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, credRole, error) {
 	c.mu.Lock()
-	roleARN, known := c.roles[credID]
+	role, known := c.roles[credID]
 	if !known {
 		c.mu.Unlock()
-		return stsauth.Credentials{}, "", errUnknownCredID
+		return stsauth.Credentials{}, credRole{}, errUnknownCredID
 	}
-	if e, ok := c.cache[credID]; ok && e.roleARN == roleARN && credValid(e.creds, credRefreshMargin) {
+	if e, ok := c.cache[credID]; ok && e.roleARN == role.roleARN && credValid(e.creds, credRefreshMargin) {
 		c.mu.Unlock()
-		return e.creds, roleARN, nil
+		return e.creds, role, nil
 	}
 	c.mu.Unlock()
 
-	creds, err := c.assume(ctx, roleARN, sessionName(credID))
+	creds, err := c.assume(ctx, role.roleARN, sessionName(role.taskID))
 	if err != nil {
-		return stsauth.Credentials{}, "", err
+		return stsauth.Credentials{}, role, err
 	}
 
 	c.mu.Lock()
-	c.cache[credID] = credEntry{roleARN: roleARN, creds: creds}
+	c.cache[credID] = credEntry{roleARN: role.roleARN, creds: creds}
 	c.mu.Unlock()
-	return creds, roleARN, nil
+	return creds, role, nil
 }
 
 // AssumeProvider returns a CredentialsProvider that mints credentials by assuming
@@ -390,10 +397,10 @@ func credValid(c stsauth.Credentials, margin time.Duration) bool {
 	return time.Now().Add(margin).Before(c.Expiration)
 }
 
-// sessionName derives an AssumeRole RoleSessionName from credID, satisfying the
+// sessionName derives an AssumeRole RoleSessionName from the task ID, satisfying the
 // STS [\w+=,.@-]{2,64} constraint.
-func sessionName(credID string) string {
-	name := "ecs-" + credID
+func sessionName(taskID string) string {
+	name := "ecs-" + taskID
 	if len(name) > 64 {
 		name = name[:64]
 	}

spinifex/agents/ecs/agent/credendpoint_test.go

@@ -29,7 +29,7 @@ func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expiration: exp,
 	})
-	c.Register("cred-1", "arn:aws:iam::111122223333:role/task")
+	c.Register("cred-1", "task-1", "arn:aws:iam::111122223333:role/task")
 
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -51,6 +51,26 @@ func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 	}
 }
 
+// The STS session name lands in the assumed-role ARN and in STS logs, so it is
+// built from the public task ID and never from the secret credID.
+func TestCredEndpoint_SessionNameIsTaskIDNotCredID(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{})
+	var gotSession string
+	c.assume = func(ctx context.Context, roleARN, session string) (stsauth.Credentials, error) {
+		gotSession = session
+		return stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"}, nil
+	}
+	c.Register("secret-cred", "task-1", "arn:role")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/secret-cred", nil))
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rr.Code)
+	}
+	if gotSession != "ecs-task-1" {
+		t.Errorf("session name = %q, want ecs-task-1", gotSession)
+	}
+}
+
 func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
 	c, calls := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
 	rr := httptest.NewRecorder()
@@ -65,7 +85,7 @@ func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
 
 func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/v2/credentials/cred-1", nil))
 	if rr.Code != http.StatusMethodNotAllowed {
@@ -77,7 +97,7 @@ func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
 	c, calls := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
 	})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	for hit := range 3 {
 		rr := httptest.NewRecorder()
 		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -95,7 +115,7 @@ func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 	c, calls := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Minute),
 	})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	for range 2 {
 		rr := httptest.NewRecorder()
 		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -107,7 +127,7 @@ func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 
 func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	c.Deregister("cred-1")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -121,7 +141,7 @@ func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
 	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
 		return stsauth.Credentials{}, context.DeadlineExceeded
 	}
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
 	if rr.Code != http.StatusBadGateway {
@@ -133,7 +153,7 @@ func TestCredEndpoint_StartServesOverLoopback(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
 	})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	c.port = 0 // bind an ephemeral port instead of privileged 80
 	if err := c.Start(); err != nil {
 		t.Fatalf("start: %v", err)

spinifex/agents/ecs/agent/credinject_test.go

@@ -1,8 +1,12 @@
 package agent
 
 import (
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
+	"github.com/mulgadc/spinifex/internal/stsauth"
+
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 )
 
@@ -13,7 +17,7 @@ func TestTaskCredID(t *testing.T) {
 		want string
 	}{
 		{"no role", ecsv1.Assign{TaskID: "t1"}, ""},
-		{"role, no credID -> taskID", ecsv1.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, "t1"},
+		{"role, no credID -> none", ecsv1.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, ""},
 		{"role + credID", ecsv1.Assign{TaskID: "t1", CredID: "c9", TaskRoleARN: "arn:role"}, "c9"},
 	}
 	for _, tc := range cases {
@@ -23,6 +27,37 @@ func TestTaskCredID(t *testing.T) {
 	}
 }
 
+// The task ID is public (it is the task ARN suffix), so naming another task's ID
+// at the credential endpoint must not yield that task's role.
+func TestRegisterTaskCreds_TaskIDDoesNotServeCredentials(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"})
+	a := Agent{cred: c}
+	get := func(id string) int {
+		rr := httptest.NewRecorder()
+		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, credRelativeURI(id), nil))
+		return rr.Code
+	}
+
+	legacy := ecsv1.Assign{TaskID: "victim-task", TaskRoleARN: "arn:aws:iam::000000000010:role/high"}
+	if got := a.registerTaskCreds(&legacy); got != "" {
+		t.Errorf("assign without CredID registered credID %q", got)
+	}
+	if code := get("victim-task"); code != http.StatusNotFound {
+		t.Errorf("GET by task ID of CredID-less assign = %d, want 404", code)
+	}
+
+	minted := ecsv1.Assign{TaskID: "victim-2", CredID: "secret-cred", TaskRoleARN: "arn:aws:iam::000000000010:role/high"}
+	if got := a.registerTaskCreds(&minted); got != "secret-cred" {
+		t.Fatalf("registerTaskCreds = %q, want secret-cred", got)
+	}
+	if code := get("victim-2"); code != http.StatusNotFound {
+		t.Errorf("GET by task ID = %d, want 404", code)
+	}
+	if code := get("secret-cred"); code != http.StatusOK {
+		t.Errorf("GET by CredID = %d, want 200", code)
+	}
+}
+
 func TestWithCredEnv(t *testing.T) {
 	base := map[string]string{"FOO": "bar"}
 

spinifex/agents/ecs/agent/reconcile_test.go

@@ -63,7 +63,7 @@ func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
 	if !adopted["t-001"] {
 		t.Fatalf("t-001 not adopted; got %v", adopted)
 	}
-	if got := a.cred.roles["t-001"]; got != testTaskRole {
+	if got := a.cred.roles["t-001"].roleARN; got != testTaskRole {
 		t.Errorf("cred not re-registered: roles[t-001] = %q, want %q", got, testTaskRole)
 	}
 	if n := countStatus(cp.taskStates(), "t-001", ecsv1.TaskStatusRunning); n != 1 {

spinifex/agents/ecs/agent/taskrun.go

@@ -306,24 +306,24 @@ func (a *Agent) pullResolver(as *ecsv1.Assign) ctrruntime.Resolver {
 // credential endpoint and returns the credID (empty when the task has no role).
 func (a *Agent) registerTaskCreds(as *ecsv1.Assign) string {
 	credID := taskCredID(as)
+	if credID == "" && as.TaskRoleARN != "" {
+		slog.Warn("ecs-agent: task has a role but no credential ID, starting it without credentials", "task", as.TaskID, "role", as.TaskRoleARN)
+	}
 	if credID == "" || a.cred == nil {
 		return credID
 	}
-	a.cred.Register(credID, as.TaskRoleARN)
+	a.cred.Register(credID, as.TaskID, as.TaskRoleARN)
 	return credID
 }
 
 // taskCredID is the credential ID a task's containers fetch credentials under:
-// the scheduler-assigned CredID, falling back to the taskID. Empty when the task
-// carries no IAM role.
+// the scheduler-minted secret CredID. Empty when the task carries no IAM role or
+// no CredID; the public task ID is never used, since any API caller can learn it.
 func taskCredID(as *ecsv1.Assign) string {
 	if as.TaskRoleARN == "" {
 		return ""
 	}
-	if as.CredID != "" {
-		return as.CredID
-	}
-	return as.TaskID
+	return as.CredID
 }
 
 // withCredEnv returns env with AWS_CONTAINER_CREDENTIALS_RELATIVE_URI set for

spinifex/agents/lambda/agent/agent.go

@@ -0,0 +1,91 @@
+// Package agent runs the guest loop: take a request from the host link, hand
+// it to the function runtime, and report the outcome back.
+package agent
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+)
+
+const (
+	errTypeTimedOut = "Sandbox.Timedout"
+	errTypeUnknown  = "Runtime.Unknown"
+)
+
+// ErrInvocationTimedOut stops the agent after it reports a timed-out
+// invocation, because the runtime may still be running it.
+var ErrInvocationTimedOut = errors.New("invocation timed out; environment must not serve another request")
+
+// Run serves requests from link until the host asks for shutdown (nil), the
+// runtime reports an init error, an invocation times out, ctx ends, or the
+// link fails. GrantID and AttemptID are echoed to the host unchanged.
+func Run(ctx context.Context, link hostlink.Link, srv *runtimeapi.Server) error {
+	ctx, cancel := context.WithCancelCause(ctx)
+	defer cancel(nil)
+	go func() {
+		select {
+		case <-srv.InitFailed():
+			ie, _ := srv.InitError()
+			cancel(fmt.Errorf("runtime init error %q: %s", ie.Type, ie.Body))
+		case <-ctx.Done():
+		}
+	}()
+
+	for {
+		req, err := link.Next(ctx)
+		if errors.Is(err, hostlink.ErrShutdown) {
+			return nil
+		}
+		if err != nil {
+			return cause(ctx, err)
+		}
+
+		res, err := srv.Invoke(ctx, runtimeapi.Invocation{
+			RequestID:   req.RequestID,
+			FunctionARN: req.FunctionARN,
+			Deadline:    time.UnixMilli(req.DeadlineMs),
+			TraceID:     req.TraceID,
+			Payload:     req.Payload,
+		})
+		timedOut := errors.Is(err, runtimeapi.ErrDeadlineExceeded)
+		if err != nil {
+			if ctx.Err() != nil {
+				return cause(ctx, err)
+			}
+			res = failure(err)
+		}
+		resp := hostlink.Response{
+			GrantID:       req.GrantID,
+			AttemptID:     req.AttemptID,
+			RequestID:     req.RequestID,
+			Payload:       res.Payload,
+			FunctionError: res.FunctionError,
+		}
+		if err := link.Complete(ctx, resp); err != nil {
+			return cause(ctx, err)
+		}
+		if timedOut {
+			return ErrInvocationTimedOut
+		}
+	}
+}
+
+func cause(ctx context.Context, err error) error {
+	if ctx.Err() != nil {
+		return context.Cause(ctx)
+	}
+	return err
+}
+
+func failure(err error) runtimeapi.Result {
+	errType := errTypeUnknown
+	if errors.Is(err, runtimeapi.ErrDeadlineExceeded) {
+		errType = errTypeTimedOut
+	}
+	return runtimeapi.Result{Payload: runtimeapi.ErrorBody(err.Error(), errType), FunctionError: errType}
+}

spinifex/agents/lambda/agent/agent_test.go

@@ -0,0 +1,142 @@
+package agent
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+)
+
+const api = "/2018-06-01/runtime/"
+
+// fakeLink feeds scripted requests and records completions.
+type fakeLink struct {
+	reqs []hostlink.Request
+	got  chan hostlink.Response
+}
+
+var _ hostlink.Link = (*fakeLink)(nil)
+
+func (l *fakeLink) Next(ctx context.Context) (hostlink.Request, error) {
+	if len(l.reqs) == 0 {
+		return hostlink.Request{}, hostlink.ErrShutdown
+	}
+	r := l.reqs[0]
+	l.reqs = l.reqs[1:]
+	return r, nil
+}
+
+func (l *fakeLink) Complete(ctx context.Context, r hostlink.Response) error {
+	l.got <- r
+	return nil
+}
+
+// runtime polls next and answers using reply.
+func runtime(t *testing.T, base string, reply func(id string, body []byte) (path string, out string)) {
+	t.Helper()
+	go func() {
+		for {
+			resp, err := http.Get(base + api + "invocation/next")
+			if err != nil {
+				return
+			}
+			body, _ := io.ReadAll(resp.Body)
+			resp.Body.Close()
+			id := resp.Header.Get("Lambda-Runtime-Aws-Request-Id")
+			path, out := reply(id, body)
+			r, err := http.Post(base+api+"invocation/"+id+path, "application/json", strings.NewReader(out))
+			if err != nil {
+				return
+			}
+			r.Body.Close()
+		}
+	}()
+}
+
+func deadline(d time.Duration) int64 { return time.Now().Add(d).UnixMilli() }
+
+func TestRunServesRequestsAndEchoesIDs(t *testing.T) {
+	srv := runtimeapi.New()
+	ts := httptest.NewServer(srv.Handler())
+	defer ts.Close()
+	runtime(t, ts.URL, func(id string, body []byte) (string, string) {
+		if string(body) == "bad" {
+			return "/error", `{"errorMessage":"no"}`
+		}
+		return "/response", "echo:" + string(body)
+	})
+
+	link := &fakeLink{
+		got: make(chan hostlink.Response, 4),
+		reqs: []hostlink.Request{
+			{GrantID: "g1", AttemptID: "a1", RequestID: "r1", FunctionARN: "arn", DeadlineMs: deadline(2 * time.Second), Payload: []byte("one")},
+			{GrantID: "g2", AttemptID: "a2", RequestID: "r2", FunctionARN: "arn", DeadlineMs: deadline(2 * time.Second), Payload: []byte("bad")},
+		},
+	}
+	require.NoError(t, Run(context.Background(), link, srv))
+
+	r1, r2 := <-link.got, <-link.got
+	assert.Equal(t, hostlink.Response{GrantID: "g1", AttemptID: "a1", RequestID: "r1", Payload: []byte("echo:one")}, r1)
+	assert.Equal(t, "g2", r2.GrantID)
+	assert.Equal(t, "a2", r2.AttemptID)
+	assert.Equal(t, "Unhandled", r2.FunctionError)
+}
+
+func TestRunReportsTimeoutAndStops(t *testing.T) {
+	srv := runtimeapi.New() // no runtime polling: the invocation expires
+	link := &fakeLink{
+		got: make(chan hostlink.Response, 2),
+		reqs: []hostlink.Request{
+			{GrantID: "g", AttemptID: "a", RequestID: "r", DeadlineMs: deadline(30 * time.Millisecond)},
+			{GrantID: "g2", AttemptID: "a2", RequestID: "r2", DeadlineMs: deadline(2 * time.Second)},
+		},
+	}
+	require.ErrorIs(t, Run(context.Background(), link, srv), ErrInvocationTimedOut)
+	r := <-link.got
+	assert.Equal(t, "Sandbox.Timedout", r.FunctionError)
+	assert.Equal(t, "g", r.GrantID)
+	assert.Len(t, link.reqs, 1, "the next request must not be taken")
+	assert.Empty(t, link.got, "nothing else is reported")
+}
+
+// blockLink blocks in Next until ctx ends.
+type blockLink struct{ fakeLink }
+
+func (blockLink) Next(ctx context.Context) (hostlink.Request, error) {
+	<-ctx.Done()
+	return hostlink.Request{}, ctx.Err()
+}
+
+func TestRunStopsOnInitError(t *testing.T) {
+	srv := runtimeapi.New()
+	ts := httptest.NewServer(srv.Handler())
+	defer ts.Close()
+	r, err := http.Post(ts.URL+api+"init/error", "application/json", strings.NewReader(`{"errorMessage":"x"}`))
+	require.NoError(t, err)
+	r.Body.Close()
+
+	err = Run(context.Background(), &blockLink{}, srv)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "init error")
+}
+
+type errLink struct{ fakeLink }
+
+func (errLink) Next(context.Context) (hostlink.Request, error) {
+	return hostlink.Request{}, errors.New("link down")
+}
+
+func TestRunReturnsLinkError(t *testing.T) {
+	err := Run(context.Background(), &errLink{}, runtimeapi.New())
+	require.EqualError(t, err, "link down")
+}

spinifex/agents/lambda/hostlink/link.go

@@ -0,0 +1,41 @@
+// Package hostlink is the guest's boundary to host control: an authenticated
+// TCP connection under per-generation mutual TLS. The message protocol over it
+// is not yet chosen; implementations of Link are carriers.
+package hostlink
+
+import (
+	"context"
+	"errors"
+)
+
+// ErrShutdown is returned by Link.Next when the host asks the agent to stop.
+var ErrShutdown = errors.New("hostlink: host requested shutdown")
+
+// Request is one invocation from the host. GrantID and AttemptID are opaque
+// to the agent and are echoed unchanged in the Response.
+type Request struct {
+	GrantID     string `json:"grant_id"`
+	AttemptID   string `json:"attempt_id"`
+	RequestID   string `json:"request_id"`
+	FunctionARN string `json:"function_arn"`
+	TraceID     string `json:"trace_id,omitempty"`
+	DeadlineMs  int64  `json:"deadline_ms"`
+	Payload     []byte `json:"payload"`
+}
+
+// Response is the outcome of a Request. FunctionError is empty on success.
+type Response struct {
+	GrantID       string `json:"grant_id"`
+	AttemptID     string `json:"attempt_id"`
+	RequestID     string `json:"request_id"`
+	Payload       []byte `json:"payload"`
+	FunctionError string `json:"function_error,omitempty"`
+}
+
+// Link carries invocations from the host to the guest and results back.
+type Link interface {
+	// Next blocks for the next request, or returns ErrShutdown.
+	Next(ctx context.Context) (Request, error)
+	// Complete reports the outcome of a request.
+	Complete(ctx context.Context, resp Response) error
+}

spinifex/agents/lambda/hostlink/test_stream.go

@@ -0,0 +1,163 @@
+//go:build lambdatestlink
+
+package hostlink
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"sync"
+)
+
+// maxFrame bounds one frame: a 6 MiB payload as base64 plus headroom.
+const maxFrame = (6<<20+2)/3*4 + 64<<10
+
+const (
+	typeInvoke   = "invoke"
+	typeShutdown = "shutdown"
+	typeResult   = "result"
+)
+
+// ErrFrameTooLarge is returned for a frame beyond the size bound.
+var ErrFrameTooLarge = errors.New("hostlink: frame too large")
+
+type frame struct {
+	Type     string    `json:"type"`
+	Request  *Request  `json:"request,omitempty"`
+	Response *Response `json:"response,omitempty"`
+}
+
+// TestStream is a test-only carrier over an io.ReadWriter, framing JSON with
+// a 4-byte big-endian length. It is not the guest protocol.
+type TestStream struct {
+	rw  io.ReadWriter
+	in  chan frame
+	wmu sync.Mutex
+
+	err error // set before in is closed
+}
+
+var _ Link = (*TestStream)(nil)
+
+// NewTestStream starts reading frames from rw.
+func NewTestStream(rw io.ReadWriter) *TestStream {
+	s := &TestStream{rw: rw, in: make(chan frame)}
+	go s.readLoop()
+	return s
+}
+
+func (s *TestStream) readLoop() {
+	defer close(s.in)
+	for {
+		f, err := readFrame(s.rw)
+		if err != nil {
+			s.err = err
+			return
+		}
+		s.in <- f
+	}
+}
+
+func (s *TestStream) recv(ctx context.Context) (frame, error) {
+	select {
+	case f, ok := <-s.in:
+		if !ok {
+			return frame{}, s.err
+		}
+		return f, nil
+	case <-ctx.Done():
+		return frame{}, ctx.Err()
+	}
+}
+
+// Next implements Link.
+func (s *TestStream) Next(ctx context.Context) (Request, error) {
+	f, err := s.recv(ctx)
+	if err != nil {
+		return Request{}, err
+	}
+	switch f.Type {
+	case typeShutdown:
+		return Request{}, ErrShutdown
+	case typeInvoke:
+		if f.Request == nil {
+			return Request{}, errors.New("hostlink: invoke frame without request")
+		}
+		return *f.Request, nil
+	default:
+		return Request{}, fmt.Errorf("hostlink: unexpected frame type %q", f.Type)
+	}
+}
+
+// Complete implements Link.
+func (s *TestStream) Complete(ctx context.Context, resp Response) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	return s.write(frame{Type: typeResult, Response: &resp})
+}
+
+// HostSend writes an invoke frame, as the host side of the carrier.
+func (s *TestStream) HostSend(req Request) error {
+	return s.write(frame{Type: typeInvoke, Request: &req})
+}
+
+// HostShutdown writes a shutdown frame, as the host side of the carrier.
+func (s *TestStream) HostShutdown() error {
+	return s.write(frame{Type: typeShutdown})
+}
+
+// HostResult reads a result frame, as the host side of the carrier.
+func (s *TestStream) HostResult(ctx context.Context) (Response, error) {
+	f, err := s.recv(ctx)
+	if err != nil {
+		return Response{}, err
+	}
+	if f.Type != typeResult || f.Response == nil {
+		return Response{}, fmt.Errorf("hostlink: unexpected frame type %q", f.Type)
+	}
+	return *f.Response, nil
+}
+
+func (s *TestStream) write(f frame) error {
+	b, err := json.Marshal(f)
+	if err != nil {
+		return err
+	}
+	if len(b) > maxFrame {
+		return ErrFrameTooLarge
+	}
+	buf := make([]byte, 4+len(b))
+	binary.BigEndian.PutUint32(buf, uint32(len(b)))
+	copy(buf[4:], b)
+	s.wmu.Lock()
+	defer s.wmu.Unlock()
+	_, err = s.rw.Write(buf)
+	return err
+}
+
+func readFrame(r io.Reader) (frame, error) {
+	var hdr [4]byte
+	if _, err := io.ReadFull(r, hdr[:]); err != nil {
+		return frame{}, err
+	}
+	n := binary.BigEndian.Uint32(hdr[:])
+	if n > maxFrame {
+		return frame{}, ErrFrameTooLarge
+	}
+	body := make([]byte, n)
+	if _, err := io.ReadFull(r, body); err != nil {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		return frame{}, err
+	}
+	var f frame
+	if err := json.Unmarshal(body, &f); err != nil {
+		return frame{}, fmt.Errorf("hostlink: bad frame: %w", err)
+	}
+	return f, nil
+}

spinifex/agents/lambda/hostlink/test_stream_test.go

@@ -0,0 +1,73 @@
+//go:build lambdatestlink
+
+package hostlink
+
+import (
+	"context"
+	"encoding/binary"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// pair returns the guest carrier and the raw host end of a pipe.
+func pair(t *testing.T) (*TestStream, net.Conn) {
+	t.Helper()
+	g, h := net.Pipe()
+	t.Cleanup(func() { g.Close(); h.Close() })
+	return NewTestStream(g), h
+}
+
+func TestRoundTrip(t *testing.T) {
+	guest, host := pair(t)
+	hostEnd := NewTestStream(host)
+	req := Request{GrantID: "g", AttemptID: "a", RequestID: "r", FunctionARN: "arn", TraceID: "t", DeadlineMs: 42, Payload: []byte{0, 1, 2, 255}}
+	go func() { _ = hostEnd.HostSend(req) }()
+
+	got, err := guest.Next(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, req, got)
+
+	resp := Response{GrantID: "g", AttemptID: "a", RequestID: "r", Payload: []byte("out"), FunctionError: "Unhandled"}
+	go func() { _ = guest.Complete(context.Background(), resp) }()
+	gotResp, err := hostEnd.HostResult(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, resp, gotResp)
+}
+
+func TestOversizeFrameRefused(t *testing.T) {
+	guest, host := pair(t)
+	go func() {
+		var hdr [4]byte
+		binary.BigEndian.PutUint32(hdr[:], maxFrame+1)
+		_, _ = host.Write(hdr[:])
+	}()
+	_, err := guest.Next(context.Background())
+	require.ErrorIs(t, err, ErrFrameTooLarge)
+
+	err = guest.Complete(context.Background(), Response{Payload: make([]byte, maxFrame)})
+	require.ErrorIs(t, err, ErrFrameTooLarge)
+}
+
+func TestTruncatedFrameErrors(t *testing.T) {
+	guest, host := pair(t)
+	go func() {
+		var hdr [4]byte
+		binary.BigEndian.PutUint32(hdr[:], 100)
+		_, _ = host.Write(append(hdr[:], []byte(`{"type"`)...))
+		host.Close()
+	}()
+	_, err := guest.Next(context.Background())
+	require.Error(t, err)
+	assert.NotErrorIs(t, err, ErrShutdown)
+}
+
+func TestShutdownSurfaced(t *testing.T) {
+	guest, host := pair(t)
+	hostEnd := NewTestStream(host)
+	go func() { _ = hostEnd.HostShutdown() }()
+	_, err := guest.Next(context.Background())
+	require.ErrorIs(t, err, ErrShutdown)
+}

spinifex/agents/lambda/hostlink/tls.go

@@ -0,0 +1,115 @@
+package hostlink
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/pki"
+)
+
+// FwCfgDir is where the guest kernel exposes the executor's fw_cfg entries.
+const FwCfgDir = "/sys/firmware/qemu_fw_cfg/by_name/" + lambdav1.FwCfgPrefix
+
+// Identity is the guest's material for one environment generation.
+type Identity struct {
+	Cert     tls.Certificate
+	HostCert *x509.Certificate
+	Scope    lambdav1.Scope
+	HostAddr string
+}
+
+// LoadIdentity reads the executor's fw_cfg entries from dir.
+func LoadIdentity(dir string) (Identity, error) {
+	read := func(name string) ([]byte, error) {
+		b, err := os.ReadFile(filepath.Join(dir, name, "raw"))
+		if err != nil {
+			return nil, fmt.Errorf("hostlink: read %s: %w", name, err)
+		}
+		return b, nil
+	}
+	var id Identity
+	keyPEM, err := read(lambdav1.EntryGuestKey)
+	if err != nil {
+		return id, err
+	}
+	certPEM, err := read(lambdav1.EntryGuestCert)
+	if err != nil {
+		return id, err
+	}
+	hostPEM, err := read(lambdav1.EntryHostCert)
+	if err != nil {
+		return id, err
+	}
+	scope, err := read(lambdav1.EntryScope)
+	if err != nil {
+		return id, err
+	}
+	addr, err := read(lambdav1.EntryHostAddr)
+	if err != nil {
+		return id, err
+	}
+
+	if id.Cert, err = tls.X509KeyPair(certPEM, keyPEM); err != nil {
+		return Identity{}, fmt.Errorf("hostlink: guest key pair: %w", err)
+	}
+	blk, rest := pem.Decode(hostPEM)
+	if blk == nil || blk.Type != "CERTIFICATE" || len(strings.TrimSpace(string(rest))) != 0 {
+		return Identity{}, errors.New("hostlink: host certificate: want exactly one PEM certificate")
+	}
+	if id.HostCert, err = x509.ParseCertificate(blk.Bytes); err != nil {
+		return Identity{}, fmt.Errorf("hostlink: host certificate: %w", err)
+	}
+	if id.Scope, err = lambdav1.ParseScope(strings.TrimSpace(string(scope))); err != nil {
+		return Identity{}, err
+	}
+	id.HostAddr = strings.TrimSpace(string(addr))
+	if _, _, err := net.SplitHostPort(id.HostAddr); err != nil {
+		return Identity{}, fmt.Errorf("hostlink: host address: %w", err)
+	}
+	return id, nil
+}
+
+// ClientConfig is the guest's TLS 1.3 configuration. The host has no DNS name,
+// so VerifyConnection checks the pinned host certificate and its URI SAN.
+func (id Identity) ClientConfig() *tls.Config {
+	return &tls.Config{
+		MinVersion:         tls.VersionTLS13,
+		MaxVersion:         tls.VersionTLS13,
+		Certificates:       []tls.Certificate{id.Cert},
+		NextProtos:         []string{lambdav1.ALPNv1},
+		InsecureSkipVerify: true, //nolint:gosec // VerifyConnection performs full validation against the pinned host certificate.
+		VerifyConnection: func(cs tls.ConnectionState) error {
+			if cs.NegotiatedProtocol != lambdav1.ALPNv1 {
+				return fmt.Errorf("hostlink: application protocol %q, want %q", cs.NegotiatedProtocol, lambdav1.ALPNv1)
+			}
+			return pki.VerifyPinned(cs.PeerCertificates, id.HostCert, x509.ExtKeyUsageServerAuth, id.Scope.URI(lambdav1.RoleHost))
+		},
+	}
+}
+
+// Dial opens the authenticated control connection to the host. Under TLS 1.3
+// the host may still refuse the guest's certificate after Dial returns; that
+// surfaces on the first read, so the caller must not treat Dial as acceptance.
+func Dial(ctx context.Context, id Identity) (*tls.Conn, error) {
+	d := tls.Dialer{NetDialer: &net.Dialer{Timeout: 10 * time.Second}, Config: id.ClientConfig()}
+	c, err := d.DialContext(ctx, "tcp", id.HostAddr)
+	if err != nil {
+		return nil, fmt.Errorf("hostlink: dial host %s: %w", id.HostAddr, err)
+	}
+	tc, ok := c.(*tls.Conn)
+	if !ok {
+		_ = c.Close()
+		return nil, fmt.Errorf("hostlink: dial host %s: unexpected connection type %T", id.HostAddr, c)
+	}
+	return tc, nil
+}

spinifex/agents/lambda/hostlink/tls_test.go

@@ -0,0 +1,291 @@
+package hostlink
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"net/url"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/pki"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	gen1 lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-1"
+	gen2 lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-2"
+)
+
+type testIdent struct {
+	cert    tls.Certificate
+	leaf    *x509.Certificate
+	certPEM []byte
+	keyPEM  []byte
+}
+
+func newTestIdent(t *testing.T, scope lambdav1.Scope, role string) testIdent {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	u, err := url.Parse(scope.URI(role))
+	require.NoError(t, err)
+	usage := x509.ExtKeyUsageServerAuth
+	if role == lambdav1.RoleGuest {
+		usage = x509.ExtKeyUsageClientAuth
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: role},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{usage},
+		BasicConstraintsValid: true, URIs: []*url.URL{u},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	require.NoError(t, err)
+	kder, err := x509.MarshalECPrivateKey(key)
+	require.NoError(t, err)
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: kder})
+	c, err := tls.X509KeyPair(certPEM, keyPEM)
+	require.NoError(t, err)
+	leaf, err := x509.ParseCertificate(der)
+	require.NoError(t, err)
+	return testIdent{cert: c, leaf: leaf, certPEM: certPEM, keyPEM: keyPEM}
+}
+
+// hostConfig is the host end as the executor will run it: it pins one guest
+// certificate and checks its URI SAN with the shared pki.VerifyPinned.
+func hostConfig(host testIdent, pinGuest *x509.Certificate, scope lambdav1.Scope) *tls.Config {
+	return &tls.Config{
+		MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{host.cert},
+		ClientAuth: tls.RequireAnyClientCert, NextProtos: []string{lambdav1.ALPNv1},
+		VerifyConnection: func(cs tls.ConnectionState) error {
+			return pki.VerifyPinned(cs.PeerCertificates, pinGuest, x509.ExtKeyUsageClientAuth, scope.URI(lambdav1.RoleGuest))
+		},
+	}
+}
+
+// serve accepts one connection with cfg, writes "ok" if the handshake
+// succeeds, and returns the address and the host's handshake result.
+func serve(t *testing.T, cfg *tls.Config) (string, <-chan error) {
+	t.Helper()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	t.Cleanup(func() { ln.Close() })
+	done := make(chan error, 1)
+	go func() {
+		raw, err := ln.Accept()
+		if err != nil {
+			done <- err
+			return
+		}
+		defer raw.Close()
+		tc := tls.Server(raw, cfg)
+		_ = tc.SetDeadline(time.Now().Add(5 * time.Second))
+		err = tc.Handshake()
+		if err == nil {
+			_, err = tc.Write([]byte("ok"))
+		}
+		done <- err
+	}()
+	return ln.Addr().String(), done
+}
+
+func guestIdentity(guest, pinHost testIdent, scope lambdav1.Scope, addr string) Identity {
+	return Identity{Cert: guest.cert, HostCert: pinHost.leaf, Scope: scope, HostAddr: addr}
+}
+
+func dialAndRead(t *testing.T, id Identity) (string, error) {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	c, err := Dial(ctx, id)
+	if err != nil {
+		return "", err
+	}
+	defer c.Close()
+	_ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
+	buf := make([]byte, 2)
+	n, err := c.Read(buf)
+	return string(buf[:n]), err
+}
+
+func TestDialMutualAuthentication(t *testing.T) {
+	host, guest := newTestIdent(t, gen1, lambdav1.RoleHost), newTestIdent(t, gen1, lambdav1.RoleGuest)
+	addr, hostErr := serve(t, hostConfig(host, guest.leaf, gen1))
+
+	got, err := dialAndRead(t, guestIdentity(guest, host, gen1, addr))
+	require.NoError(t, err)
+	assert.Equal(t, "ok", got)
+	require.NoError(t, <-hostErr)
+}
+
+func TestDialRefusesHost(t *testing.T) {
+	guest := newTestIdent(t, gen1, lambdav1.RoleGuest)
+	pinned := newTestIdent(t, gen1, lambdav1.RoleHost)
+	cases := []struct {
+		name    string
+		host    testIdent
+		pin     testIdent
+		mutate  func(*tls.Config)
+		wantErr string
+	}{
+		{"unpinned host certificate", newTestIdent(t, gen1, lambdav1.RoleHost), pinned, nil, "peer certificate"},
+		{"pinned certificate of a later generation", newTestIdent(t, gen2, lambdav1.RoleHost), testIdent{}, nil, "peer identity mismatch"},
+		{"guest role presented as host", newTestIdent(t, gen1, lambdav1.RoleGuest), testIdent{}, nil, "peer certificate"},
+		{"no application protocol", pinned, pinned, func(c *tls.Config) { c.NextProtos = nil }, "application protocol"},
+		{"TLS 1.2 only", pinned, pinned, func(c *tls.Config) { c.MinVersion, c.MaxVersion = tls.VersionTLS12, tls.VersionTLS12 }, "protocol version"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			pin := tc.pin
+			if pin.leaf == nil {
+				pin = tc.host // the guest pins exactly what is presented
+			}
+			cfg := hostConfig(tc.host, guest.leaf, gen1)
+			if tc.mutate != nil {
+				tc.mutate(cfg)
+			}
+			addr, _ := serve(t, cfg)
+			_, err := dialAndRead(t, guestIdentity(guest, pin, gen1, addr))
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.wantErr)
+		})
+	}
+}
+
+// newCASignedHost returns a CA and a host leaf it signed carrying the same URI
+// SAN as a legitimate host certificate.
+func newCASignedHost(t *testing.T, scope lambdav1.Scope) (ca testIdent, leaf tls.Certificate) {
+	t.Helper()
+	caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	u, err := url.Parse(scope.URI(lambdav1.RoleHost))
+	require.NoError(t, err)
+	caTmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "ca"},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true, IsCA: true, URIs: []*url.URL{u},
+	}
+	caDER, err := x509.CreateCertificate(rand.Reader, caTmpl, caTmpl, &caKey.PublicKey, caKey)
+	require.NoError(t, err)
+	caCert, err := x509.ParseCertificate(caDER)
+	require.NoError(t, err)
+
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "host"},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		URIs: []*url.URL{u},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, &key.PublicKey, caKey)
+	require.NoError(t, err)
+	return testIdent{leaf: caCert}, tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
+}
+
+// Trusting the pin as a root is not enough: a different leaf it signed, with
+// the right URI SAN, must still be refused.
+func TestDialRefusesLeafSignedByPin(t *testing.T) {
+	guest := newTestIdent(t, gen1, lambdav1.RoleGuest)
+	ca, leaf := newCASignedHost(t, gen1)
+	for name, chain := range map[string][][]byte{
+		"leaf only":    leaf.Certificate,
+		"leaf with CA": {leaf.Certificate[0], ca.leaf.Raw},
+	} {
+		t.Run(name, func(t *testing.T) {
+			cfg := hostConfig(testIdent{}, guest.leaf, gen1)
+			cfg.Certificates = []tls.Certificate{{Certificate: chain, PrivateKey: leaf.PrivateKey}}
+			addr, _ := serve(t, cfg)
+			_, err := dialAndRead(t, guestIdentity(guest, ca, gen1, addr))
+			require.Error(t, err)
+			assert.Regexp(t, "not the pinned certificate|want exactly the pinned one", err.Error())
+		})
+	}
+}
+
+func TestHostRefusalSurfacesOnFirstRead(t *testing.T) {
+	host, guest := newTestIdent(t, gen1, lambdav1.RoleHost), newTestIdent(t, gen1, lambdav1.RoleGuest)
+	other := newTestIdent(t, gen1, lambdav1.RoleGuest)
+	addr, hostErr := serve(t, hostConfig(host, other.leaf, gen1))
+
+	_, err := dialAndRead(t, guestIdentity(guest, host, gen1, addr))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "bad certificate")
+	assert.ErrorContains(t, <-hostErr, "peer certificate")
+}
+
+func TestHostRefusesGuestOfAnotherGeneration(t *testing.T) {
+	host, guest := newTestIdent(t, gen1, lambdav1.RoleHost), newTestIdent(t, gen2, lambdav1.RoleGuest)
+	addr, hostErr := serve(t, hostConfig(host, guest.leaf, gen1))
+
+	_, err := dialAndRead(t, guestIdentity(guest, host, gen1, addr))
+	require.Error(t, err)
+	assert.ErrorIs(t, <-hostErr, pki.ErrPeerIdentity)
+}
+
+func writeEntries(t *testing.T, entries map[string][]byte) string {
+	t.Helper()
+	dir := t.TempDir()
+	for name, b := range entries {
+		require.NoError(t, os.MkdirAll(filepath.Join(dir, name), 0o700))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, name, "raw"), b, 0o600))
+	}
+	return dir
+}
+
+func validEntries(t *testing.T) map[string][]byte {
+	host, guest := newTestIdent(t, gen1, lambdav1.RoleHost), newTestIdent(t, gen1, lambdav1.RoleGuest)
+	return map[string][]byte{
+		lambdav1.EntryGuestKey: guest.keyPEM, lambdav1.EntryGuestCert: guest.certPEM, lambdav1.EntryHostCert: host.certPEM,
+		lambdav1.EntryScope: []byte(string(gen1) + "\n"), lambdav1.EntryHostAddr: []byte("10.15.8.1:18443\n"),
+	}
+}
+
+func TestLoadIdentity(t *testing.T) {
+	id, err := LoadIdentity(writeEntries(t, validEntries(t)))
+	require.NoError(t, err)
+	assert.Equal(t, gen1, id.Scope)
+	assert.Equal(t, "10.15.8.1:18443", id.HostAddr)
+	require.Len(t, id.HostCert.URIs, 1)
+	assert.Equal(t, gen1.URI(lambdav1.RoleHost), id.HostCert.URIs[0].String())
+}
+
+func TestLoadIdentityRejects(t *testing.T) {
+	other := newTestIdent(t, gen1, lambdav1.RoleGuest)
+	cases := []struct {
+		name    string
+		edit    func(map[string][]byte)
+		wantErr string
+	}{
+		{"missing entry", func(e map[string][]byte) { delete(e, lambdav1.EntryHostCert) }, "read host-cert"},
+		{"key does not match certificate", func(e map[string][]byte) { e[lambdav1.EntryGuestKey] = other.keyPEM }, "guest key pair"},
+		{"two host certificates", func(e map[string][]byte) {
+			e[lambdav1.EntryHostCert] = append(e[lambdav1.EntryHostCert], e[lambdav1.EntryHostCert]...)
+		}, "exactly one PEM"},
+		{"host certificate not PEM", func(e map[string][]byte) { e[lambdav1.EntryHostCert] = []byte("nope") }, "exactly one PEM"},
+		{"short scope", func(e map[string][]byte) { e[lambdav1.EntryScope] = []byte("ap-southeast-2/000000000001/fn-1") }, "want 5 segments"},
+		{"address without port", func(e map[string][]byte) { e[lambdav1.EntryHostAddr] = []byte("10.15.8.1") }, "host address"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			e := validEntries(t)
+			tc.edit(e)
+			_, err := LoadIdentity(writeEntries(t, e))
+			assert.ErrorContains(t, err, tc.wantErr)
+		})
+	}
+}

spinifex/agents/lambda/runtimeapi/server.go

@@ -0,0 +1,285 @@
+// Package runtimeapi serves the AWS Lambda Runtime API (version 2018-06-01)
+// inside the guest, handing one invocation at a time to the function runtime.
+package runtimeapi
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// MaxResponseBytes is the largest response payload a runtime may post.
+const MaxResponseBytes = 6291456
+
+const (
+	// ErrTypeResponseTooLarge is the function error for an oversize response.
+	ErrTypeResponseTooLarge = "Function.ResponseSizeTooLarge"
+	// ErrTypeUnhandled is the function error when the runtime names no type.
+	ErrTypeUnhandled = "Unhandled"
+
+	errTypeInvalidRequestID = "InvalidRequestID"
+	errTypeTooLarge         = "RequestEntityTooLarge"
+	errTypeInvalidRequest   = "InvalidRequest"
+
+	headerRequestID = "Lambda-Runtime-Aws-Request-Id"
+	headerDeadline  = "Lambda-Runtime-Deadline-Ms"
+	headerARN       = "Lambda-Runtime-Invoked-Function-Arn"
+	headerTraceID   = "Lambda-Runtime-Trace-Id"
+	headerErrorType = "Lambda-Runtime-Function-Error-Type"
+
+	maxInitErrorBytes = 1 << 20
+)
+
+var (
+	// ErrBusy is returned by Invoke while another invocation is in flight.
+	ErrBusy = errors.New("runtimeapi: an invocation is already in flight")
+	// ErrDeadlineExceeded is returned by Invoke when the deadline passes.
+	ErrDeadlineExceeded = errors.New("runtimeapi: invocation deadline exceeded")
+	// ErrInvalidInvocation is returned by Invoke for an invocation with no request ID.
+	ErrInvalidInvocation = errors.New("runtimeapi: invocation has no request id")
+)
+
+// Invocation is one event handed to the runtime.
+type Invocation struct {
+	RequestID   string
+	FunctionARN string
+	Deadline    time.Time
+	TraceID     string
+	Payload     []byte
+}
+
+// Result is what the runtime answered. FunctionError is empty on success and
+// otherwise the error type.
+type Result struct {
+	Payload       []byte
+	FunctionError string
+}
+
+// InitError is an initialisation failure reported by the runtime.
+type InitError struct {
+	Type string
+	Body []byte
+}
+
+type flight struct {
+	inv   Invocation
+	taken bool
+	done  chan Result
+}
+
+// Server implements the Runtime API for a single in-flight invocation.
+type Server struct {
+	mu   sync.Mutex
+	cur  *flight
+	wake chan struct{}
+
+	initOnce sync.Once
+	initDone chan struct{}
+	initErr  InitError
+}
+
+// New returns a Server with no invocation in flight.
+func New() *Server {
+	return &Server{wake: make(chan struct{}, 1), initDone: make(chan struct{})}
+}
+
+// InitFailed is closed once the runtime reports an init error.
+func (s *Server) InitFailed() <-chan struct{} { return s.initDone }
+
+// InitError returns the reported init error, if any.
+func (s *Server) InitError() (InitError, bool) {
+	select {
+	case <-s.initDone:
+		return s.initErr, true
+	default:
+		return InitError{}, false
+	}
+}
+
+// Invoke hands inv to the runtime and blocks until the runtime posts a
+// response or error, the deadline passes, or ctx ends.
+func (s *Server) Invoke(ctx context.Context, inv Invocation) (Result, error) {
+	if inv.RequestID == "" {
+		return Result{}, ErrInvalidInvocation
+	}
+	f := &flight{inv: inv, done: make(chan Result, 1)}
+	s.mu.Lock()
+	if s.cur != nil {
+		s.mu.Unlock()
+		return Result{}, ErrBusy
+	}
+	s.cur = f
+	s.mu.Unlock()
+	select {
+	case s.wake <- struct{}{}:
+	default:
+	}
+
+	timer := time.NewTimer(time.Until(inv.Deadline))
+	defer timer.Stop()
+	var cause error
+	select {
+	case res := <-f.done:
+		return res, nil
+	case <-timer.C:
+		cause = ErrDeadlineExceeded
+	case <-ctx.Done():
+		cause = ctx.Err()
+	}
+
+	s.mu.Lock()
+	if s.cur == f {
+		s.cur = nil
+		s.mu.Unlock()
+		return Result{}, cause
+	}
+	s.mu.Unlock()
+	// Completed while we were giving up; the result is already queued.
+	return <-f.done, nil
+}
+
+// Handler returns the Runtime API routes.
+func (s *Server) Handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /2018-06-01/runtime/invocation/next", s.next)
+	mux.HandleFunc("POST /2018-06-01/runtime/invocation/{id}/response", s.response)
+	mux.HandleFunc("POST /2018-06-01/runtime/invocation/{id}/error", s.invocationError)
+	mux.HandleFunc("POST /2018-06-01/runtime/init/error", s.initError)
+	return mux
+}
+
+func (s *Server) next(w http.ResponseWriter, r *http.Request) {
+	for {
+		s.mu.Lock()
+		f := s.cur
+		if f != nil && !f.taken {
+			f.taken = true
+			s.mu.Unlock()
+			h := w.Header()
+			h.Set(headerRequestID, f.inv.RequestID)
+			h.Set(headerDeadline, strconv.FormatInt(f.inv.Deadline.UnixMilli(), 10))
+			h.Set(headerARN, f.inv.FunctionARN)
+			if f.inv.TraceID != "" {
+				h.Set(headerTraceID, f.inv.TraceID)
+			}
+			_, _ = w.Write(f.inv.Payload)
+			return
+		}
+		s.mu.Unlock()
+		select {
+		case <-s.wake:
+		case <-r.Context().Done():
+			return
+		}
+	}
+}
+
+// claimable reports whether id names the delivered, uncompleted invocation.
+func (s *Server) claimable(id string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.cur != nil && s.cur.taken && s.cur.inv.RequestID == id
+}
+
+// complete finishes the invocation named id, reporting whether it was live.
+func (s *Server) complete(id string, res Result) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	f := s.cur
+	if f == nil || !f.taken || f.inv.RequestID != id {
+		return false
+	}
+	s.cur = nil
+	f.done <- res
+	return true
+}
+
+func (s *Server) response(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	if !s.claimable(id) {
+		invalidID(w, id)
+		return
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, MaxResponseBytes+1))
+	if err != nil {
+		writeError(w, http.StatusBadRequest, errTypeInvalidRequest, "read body: "+err.Error())
+		return
+	}
+	if len(body) > MaxResponseBytes {
+		msg := "Response payload size exceeded maximum allowed payload size (" + strconv.Itoa(MaxResponseBytes) + " bytes)."
+		s.complete(id, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeResponseTooLarge})
+		writeError(w, http.StatusRequestEntityTooLarge, errTypeTooLarge, msg)
+		return
+	}
+	if !s.complete(id, Result{Payload: body}) {
+		invalidID(w, id)
+		return
+	}
+	writeStatus(w)
+}
+
+func (s *Server) invocationError(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	if !s.claimable(id) {
+		invalidID(w, id)
+		return
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, MaxResponseBytes))
+	if err != nil {
+		writeError(w, http.StatusBadRequest, errTypeInvalidRequest, "read body: "+err.Error())
+		return
+	}
+	errType := r.Header.Get(headerErrorType)
+	if errType == "" {
+		errType = ErrTypeUnhandled
+	}
+	if !s.complete(id, Result{Payload: body, FunctionError: errType}) {
+		invalidID(w, id)
+		return
+	}
+	writeStatus(w)
+}
+
+func (s *Server) initError(w http.ResponseWriter, r *http.Request) {
+	body, _ := io.ReadAll(io.LimitReader(r.Body, maxInitErrorBytes))
+	s.initOnce.Do(func() {
+		s.initErr = InitError{Type: r.Header.Get(headerErrorType), Body: body}
+		close(s.initDone)
+	})
+	writeStatus(w)
+}
+
+func invalidID(w http.ResponseWriter, id string) {
+	writeError(w, http.StatusBadRequest, errTypeInvalidRequestID, "Invalid request ID: "+id)
+}
+
+func writeStatus(w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusAccepted)
+	_, _ = w.Write([]byte(`{"status":"OK"}`))
+}
+
+func writeError(w http.ResponseWriter, code int, errType, msg string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(code)
+	_, _ = w.Write(ErrorBody(msg, errType))
+}
+
+type errorPayload struct {
+	ErrorMessage string `json:"errorMessage"`
+	ErrorType    string `json:"errorType"`
+}
+
+// ErrorBody renders the Lambda error document for msg and errType.
+func ErrorBody(msg, errType string) []byte {
+	b, err := json.Marshal(errorPayload{ErrorMessage: msg, ErrorType: errType})
+	if err != nil {
+		return []byte(`{"errorType":"Runtime.Unknown"}`)
+	}
+	return b
+}

spinifex/agents/lambda/runtimeapi/server_test.go

@@ -0,0 +1,292 @@
+package runtimeapi
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const base = "/2018-06-01/runtime/"
+
+type invokeOut struct {
+	res Result
+	err error
+}
+
+func start(t *testing.T) (*Server, *httptest.Server) {
+	t.Helper()
+	s := New()
+	ts := httptest.NewServer(s.Handler())
+	t.Cleanup(ts.Close)
+	return s, ts
+}
+
+func invokeAsync(s *Server, inv Invocation) <-chan invokeOut {
+	ch := make(chan invokeOut, 1)
+	go func() {
+		res, err := s.Invoke(context.Background(), inv)
+		ch <- invokeOut{res, err}
+	}()
+	return ch
+}
+
+func post(t *testing.T, url string, body []byte, hdr map[string]string) (*http.Response, []byte) {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
+	require.NoError(t, err)
+	for k, v := range hdr {
+		req.Header.Set(k, v)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(resp.Body)
+	return resp, b
+}
+
+func get(t *testing.T, url string) (*http.Response, []byte) {
+	t.Helper()
+	resp, err := http.Get(url)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(resp.Body)
+	return resp, b
+}
+
+func inv(id string, d time.Duration) Invocation {
+	return Invocation{
+		RequestID:   id,
+		FunctionARN: "arn:aws:lambda:ap-southeast-2:000000000001:function:f",
+		Deadline:    time.Now().Add(d),
+		Payload:     []byte(`{"a":1}`),
+	}
+}
+
+func errType(t *testing.T, body []byte) string {
+	t.Helper()
+	var e map[string]string
+	require.NoError(t, json.Unmarshal(body, &e))
+	assert.NotEmpty(t, e["errorMessage"])
+	return e["errorType"]
+}
+
+func TestNextResponseRoundTrip(t *testing.T) {
+	s, ts := start(t)
+	in := inv("req-1", 2*time.Second)
+	in.TraceID = "Root=1-abc"
+	out := invokeAsync(s, in)
+
+	resp, body := get(t, ts.URL+base+"invocation/next")
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, `{"a":1}`, string(body))
+	assert.Equal(t, "req-1", resp.Header.Get("Lambda-Runtime-Aws-Request-Id"))
+	assert.Equal(t, strconv.FormatInt(in.Deadline.UnixMilli(), 10), resp.Header.Get("Lambda-Runtime-Deadline-Ms"))
+	assert.Equal(t, in.FunctionARN, resp.Header.Get("Lambda-Runtime-Invoked-Function-Arn"))
+	assert.Equal(t, "Root=1-abc", resp.Header.Get("Lambda-Runtime-Trace-Id"))
+
+	r, _ := post(t, ts.URL+base+"invocation/req-1/response", []byte(`{"sum":3}`), nil)
+	assert.Equal(t, http.StatusAccepted, r.StatusCode)
+
+	got := <-out
+	require.NoError(t, got.err)
+	assert.Equal(t, `{"sum":3}`, string(got.res.Payload))
+	assert.Empty(t, got.res.FunctionError)
+}
+
+func TestTraceHeaderOmittedWhenUnset(t *testing.T) {
+	s, ts := start(t)
+	invokeAsync(s, inv("req-1", time.Second))
+	resp, _ := get(t, ts.URL+base+"invocation/next")
+	_, present := resp.Header["Lambda-Runtime-Trace-Id"]
+	assert.False(t, present)
+}
+
+func TestErrorPostKeepsBodyAndType(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+
+	body := []byte(`{"errorMessage":"boom","errorType":"Custom"}`)
+	r, _ := post(t, ts.URL+base+"invocation/req-1/error", body, map[string]string{"Lambda-Runtime-Function-Error-Type": "Custom"})
+	assert.Equal(t, http.StatusAccepted, r.StatusCode)
+
+	got := <-out
+	require.NoError(t, got.err)
+	assert.Equal(t, "Custom", got.res.FunctionError)
+	assert.JSONEq(t, string(body), string(got.res.Payload))
+}
+
+func TestErrorPostWithoutTypeIsUnhandled(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+	post(t, ts.URL+base+"invocation/req-1/error", []byte(`{}`), nil)
+	assert.Equal(t, "Unhandled", (<-out).res.FunctionError)
+}
+
+func TestUnknownAndRepeatedIDRefused(t *testing.T) {
+	s, ts := start(t)
+
+	r, b := post(t, ts.URL+base+"invocation/nope/response", []byte("x"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+	assert.Equal(t, "InvalidRequestID", errType(t, b))
+
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+	post(t, ts.URL+base+"invocation/req-1/response", []byte("ok"), nil)
+	<-out
+
+	r, b = post(t, ts.URL+base+"invocation/req-1/response", []byte("again"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+	assert.Equal(t, "InvalidRequestID", errType(t, b))
+	r, _ = post(t, ts.URL+base+"invocation/req-1/error", []byte("{}"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+}
+
+func TestResponseNotAcceptedBeforeDelivery(t *testing.T) {
+	s, ts := start(t)
+	invokeAsync(s, inv("req-1", time.Second))
+	r, _ := post(t, ts.URL+base+"invocation/req-1/response", []byte("early"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+}
+
+func TestOversizeResponse(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 5*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+
+	r, b := post(t, ts.URL+base+"invocation/req-1/response", make([]byte, MaxResponseBytes+1), nil)
+	assert.Equal(t, http.StatusRequestEntityTooLarge, r.StatusCode)
+	assert.Equal(t, "RequestEntityTooLarge", errType(t, b))
+
+	got := <-out
+	require.NoError(t, got.err)
+	assert.Equal(t, "Function.ResponseSizeTooLarge", got.res.FunctionError)
+	assert.Equal(t, "Function.ResponseSizeTooLarge", errType(t, got.res.Payload))
+}
+
+func TestResponseAtLimitAccepted(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 5*time.Second))
+	get(t, ts.URL+base+"invocation/next")
+	r, _ := post(t, ts.URL+base+"invocation/req-1/response", make([]byte, MaxResponseBytes), nil)
+	assert.Equal(t, http.StatusAccepted, r.StatusCode)
+	got := <-out
+	require.NoError(t, got.err)
+	assert.Len(t, got.res.Payload, MaxResponseBytes)
+}
+
+func TestDeadlineExpiryRefusesLateResponse(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 40*time.Millisecond))
+	get(t, ts.URL+base+"invocation/next")
+
+	got := <-out
+	require.ErrorIs(t, got.err, ErrDeadlineExceeded)
+
+	r, b := post(t, ts.URL+base+"invocation/req-1/response", []byte("late"), nil)
+	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
+	assert.Equal(t, "InvalidRequestID", errType(t, b))
+
+	// The slot is free again for the next invocation.
+	out = invokeAsync(s, inv("req-2", 2*time.Second))
+	resp, _ := get(t, ts.URL+base+"invocation/next")
+	assert.Equal(t, "req-2", resp.Header.Get("Lambda-Runtime-Aws-Request-Id"))
+	post(t, ts.URL+base+"invocation/req-2/response", []byte("ok"), nil)
+	require.NoError(t, (<-out).err)
+}
+
+func TestConcurrentInvokeRefused(t *testing.T) {
+	s, ts := start(t)
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	require.Eventually(t, func() bool {
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		return s.cur != nil
+	}, time.Second, time.Millisecond)
+
+	_, err := s.Invoke(context.Background(), inv("req-2", time.Second))
+	require.ErrorIs(t, err, ErrBusy)
+
+	get(t, ts.URL+base+"invocation/next")
+	post(t, ts.URL+base+"invocation/req-1/response", []byte("ok"), nil)
+	require.NoError(t, (<-out).err)
+}
+
+func TestInvokeContextCancel(t *testing.T) {
+	s, _ := start(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() {
+		_, err := s.Invoke(ctx, inv("req-1", time.Minute))
+		done <- err
+	}()
+	cancel()
+	require.ErrorIs(t, <-done, context.Canceled)
+}
+
+func TestNextBlocksUntilInvoke(t *testing.T) {
+	s, ts := start(t)
+	type got struct {
+		resp *http.Response
+		body []byte
+	}
+	ch := make(chan got, 1)
+	go func() {
+		resp, err := http.Get(ts.URL + base + "invocation/next") //nolint:noctx // test client
+		if err != nil {
+			ch <- got{}
+			return
+		}
+		defer resp.Body.Close()
+		body, _ := io.ReadAll(resp.Body)
+		ch <- got{resp, body}
+	}()
+
+	select {
+	case <-ch:
+		t.Fatal("next returned before any invocation")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	out := invokeAsync(s, inv("req-1", 2*time.Second))
+	g := <-ch
+	require.NotNil(t, g.resp, "next request failed")
+	assert.Equal(t, "req-1", g.resp.Header.Get("Lambda-Runtime-Aws-Request-Id"))
+	post(t, ts.URL+base+"invocation/req-1/response", []byte("ok"), nil)
+	require.NoError(t, (<-out).err)
+}
+
+func TestInitErrorRecorded(t *testing.T) {
+	s, ts := start(t)
+	_, ok := s.InitError()
+	require.False(t, ok)
+
+	r, _ := post(t, ts.URL+base+"init/error", []byte(`{"errorMessage":"bad"}`), map[string]string{"Lambda-Runtime-Function-Error-Type": "Runtime.InitError"})
+	assert.Equal(t, http.StatusAccepted, r.StatusCode)
+
+	select {
+	case <-s.InitFailed():
+	default:
+		t.Fatal("InitFailed not closed")
+	}
+	ie, ok := s.InitError()
+	require.True(t, ok)
+	assert.Equal(t, "Runtime.InitError", ie.Type)
+	assert.JSONEq(t, `{"errorMessage":"bad"}`, string(ie.Body))
+}
+
+func TestInvokeRequiresRequestID(t *testing.T) {
+	s := New()
+	_, err := s.Invoke(context.Background(), Invocation{Deadline: time.Now().Add(time.Second)})
+	require.ErrorIs(t, err, ErrInvalidInvocation)
+}

spinifex/domains/acm/keyalgorithm_test.go

@@ -152,3 +152,30 @@ func TestImportCertificate_SerialZeroRejected(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorValidationException, code)
 	assert.Equal(t, "The serial number in the certificate is not supported by ACM.", msg)
 }
+
+func TestImportCertificate_ExpiredRejected(t *testing.T) {
+	t.Parallel()
+	svc := setupACMService(t)
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	require.NoError(t, err)
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(time.Now().UnixNano()),
+		Subject:      pkix.Name{CommonName: "expired.example.com"},
+		NotBefore:    time.Now().Add(-48 * time.Hour),
+		NotAfter:     time.Now().Add(-24 * time.Hour),
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	require.NoError(t, err)
+	_, err = svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
+		Certificate: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}),
+		PrivateKey:  pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}),
+	}, testAccountID)
+	code, msg, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorValidationException, code)
+	assert.Equal(t, "The certificate is expired. You can Import a certificate only during its validity period.", msg)
+
+	list, err := svc.ListCertificates(context.Background(), &acm.ListCertificatesInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, list.CertificateSummaryList)
+}

spinifex/domains/acm/service_impl.go

@@ -223,10 +223,13 @@ func (s *ACMServiceImpl) ImportCertificate(ctx context.Context, input *acm.Impor
 	if leaf.SerialNumber.Sign() == 0 {
 		return nil, awserrors.Errorf(awserrors.ErrorValidationException, "The serial number in the certificate is not supported by ACM.")
 	}
-
 	certArn := aws.StringValue(input.CertificateArn)
-	var inUseBy []string
 	now := time.Now().UTC()
+	if certArn == "" && now.After(leaf.NotAfter) {
+		return nil, awserrors.Errorf(awserrors.ErrorValidationException, "The certificate is expired. You can Import a certificate only during its validity period.")
+	}
+
+	var inUseBy []string
 	createdAt := now
 	tags := tagsToMap(input.Tags)
 	reimport := certArn != ""

spinifex/domains/ec2/awsapi/actions.go

@@ -134,6 +134,9 @@ func marshalEC2Response(action string, output any) ([]byte, error) {
 
 var ec2Actions = map[string]ec2Action{
 	"DescribeInstances": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstancesInput, c *call, accountID string) (any, error) {
+		if err := ec2instanceapi.ValidateDescribeInstancesInput(input); err != nil {
+			return nil, err
+		}
 		out, err := ec2instanceapi.DescribeInstancesChecked(ctx, input, c.NATS, 0, c.NodeIDs, accountID)
 		if err != nil {
 			return out, err
@@ -257,6 +260,9 @@ var ec2Actions = map[string]ec2Action{
 		}
 	}),
 	"DescribeImages": ec2Handler(func(ctx context.Context, input *ec2.DescribeImagesInput, c *call, accountID string) (any, error) {
+		if err := ec2imageapi.ValidateDescribeImagesInput(input); err != nil {
+			return nil, err
+		}
 		return ec2imageapi.DescribeImages(ctx, input, c.NATS, accountID)
 	}),
 	"CreateImage": ec2Handler(func(ctx context.Context, input *ec2.CreateImageInput, c *call, accountID string) (any, error) {
@@ -287,6 +293,9 @@ var ec2Actions = map[string]ec2Action{
 		return ec2zoneapi.DescribeAvailabilityZones(input, c.Region, c.AZ)
 	}),
 	"DescribeVolumes": ec2Handler(func(ctx context.Context, input *ec2.DescribeVolumesInput, c *call, accountID string) (any, error) {
+		if err := ec2volumeapi.ValidateDescribeVolumesInput(input); err != nil {
+			return nil, err
+		}
 		return ec2volumeapi.DescribeVolumes(ctx, input, c.NATS, accountID)
 	}),
 	"ModifyVolume": ec2Handler(func(ctx context.Context, input *ec2.ModifyVolumeInput, c *call, accountID string) (any, error) {

spinifex/domains/ec2/awsapi/actions_test.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -30,3 +31,27 @@ func TestImportKeyPair_Base64PaddingWorkaround(t *testing.T) {
 	assert.NotContains(t, q["PublicKeyMaterial"], "%3D",
 		"Expected no URL-encoded padding remaining")
 }
+
+// A well-prefixed ID of the wrong shape is refused before any service is
+// asked, so it can never come back as NotFound.
+func TestDescribe_MalformedIDRejectedBeforeLookup(t *testing.T) {
+	tests := []struct {
+		action, param, id, code, msg string
+	}{
+		{"DescribeInstances", "InstanceId.1", "i-zzz", awserrors.ErrorInvalidInstanceIDMalformed, `Invalid id: "i-zzz"`},
+		{"DescribeImages", "ImageId.1", "ami-zzz", awserrors.ErrorInvalidAMIIDMalformed, `Invalid id: "ami-zzz"`},
+		{"DescribeVolumes", "VolumeId.1", "vol-zzz", awserrors.ErrorInvalidParameterValue, "Value (vol-zzz) for parameter volumes is invalid. Expected: 'vol-...'."},
+	}
+	for _, tt := range tests {
+		t.Run(tt.action, func(t *testing.T) {
+			h := ec2Actions[tt.action]
+			input, err := h.parse(map[string]string{tt.param: tt.id})
+			require.NoError(t, err)
+			_, err = h.dispatch(t.Context(), tt.action, input, &call{Handler: NewHandler(Deps{})}, "123456789012")
+			code, msg, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok)
+			assert.Equal(t, tt.code, code)
+			assert.Equal(t, tt.msg, msg)
+		})
+	}
+}

spinifex/domains/ec2/awsapi/image/DescribeImages.go

@@ -5,6 +5,7 @@ package image
 import (
 	"context"
 	"errors"
+	"regexp"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -13,8 +14,12 @@ import (
 	"github.com/nats-io/nats.go"
 )
 
-// ValidateDescribeImagesInput accepts a nil input and rejects any ImageIds entry without the ami-
-// prefix with InvalidAMIID.Malformed.
+// imageIDSuffixPattern is the AWS AMI ID shape after ami-: 8 (legacy) or 17
+// lowercase hex characters.
+var imageIDSuffixPattern = regexp.MustCompile(`^([0-9a-f]{8}|[0-9a-f]{17})$`)
+
+// ValidateDescribeImagesInput accepts a nil input and rejects any ImageIds entry that is not
+// ami- followed by 8 or 17 lowercase hex characters with InvalidAMIID.Malformed.
 func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
 	if input == nil {
 		return nil
@@ -22,9 +27,16 @@ func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
 
 	if input.ImageIds != nil {
 		for _, imageId := range input.ImageIds {
-			if imageId != nil && !strings.HasPrefix(*imageId, "ami-") {
+			if imageId == nil {
+				continue
+			}
+			suffix, ok := strings.CutPrefix(*imageId, "ami-")
+			if !ok {
 				return errors.New(awserrors.ErrorInvalidAMIIDMalformed)
 			}
+			if !imageIDSuffixPattern.MatchString(suffix) {
+				return awserrors.Errorf(awserrors.ErrorInvalidAMIIDMalformed, "Invalid id: %q", *imageId)
+			}
 		}
 	}
 
@@ -34,11 +46,6 @@ func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
 // DescribeImages implements the EC2 DescribeImages action, returning the AMIs visible to
 // accountID from the NATS image service.
 func DescribeImages(ctx context.Context, input *ec2.DescribeImagesInput, natsConn *nats.Conn, accountID string) (output ec2.DescribeImagesOutput, err error) {
-	err = ValidateDescribeImagesInput(input)
-	if err != nil {
-		return output, err
-	}
-
 	imageService := ec2image.NewNATSImageService(natsConn, 0)
 	result, err := imageService.DescribeImages(ctx, input, accountID)
 	if err != nil {

spinifex/domains/ec2/awsapi/image/DescribeImages_test.go

@@ -53,8 +53,8 @@ func TestValidateDescribeImagesInput(t *testing.T) {
 			name: "MultipleValidImageIds",
 			input: &ec2.DescribeImagesInput{
 				ImageIds: []*string{
-					aws.String("ami-111"),
-					aws.String("ami-222"),
+					aws.String("ami-1a2b3c4d"),
+					aws.String("ami-0123456789abcdef1"),
 				},
 			},
 			wantErr: false,
@@ -63,7 +63,7 @@ func TestValidateDescribeImagesInput(t *testing.T) {
 			name: "MixedValidAndInvalidImageIds",
 			input: &ec2.DescribeImagesInput{
 				ImageIds: []*string{
-					aws.String("ami-valid"),
+					aws.String("ami-0123456789abcdef0"),
 					aws.String("invalid-id"),
 				},
 			},
@@ -85,11 +85,24 @@ func TestValidateDescribeImagesInput(t *testing.T) {
 			err := ValidateDescribeImagesInput(tt.input)
 
 			if tt.wantErr {
-				assert.Error(t, err)
-				assert.Equal(t, tt.errMsg, err.Error())
+				code, ok := awserrors.ResolveErrorCode(err)
+				assert.True(t, ok)
+				assert.Equal(t, tt.errMsg, code)
 			} else {
 				assert.NoError(t, err)
 			}
 		})
 	}
 }
+
+// AWS answers a well-prefixed ID whose suffix is not 8 or 17 lowercase hex
+// characters with Malformed, not NotFound.
+func TestValidateDescribeImagesInput_MalformedSuffix(t *testing.T) {
+	for _, id := range []string{"ami-zzz", "ami-0123456789ABCDEF0", "ami-0123456789abcdef", "ami-"} {
+		err := ValidateDescribeImagesInput(&ec2.DescribeImagesInput{ImageIds: []*string{aws.String(id)}})
+		code, msg, ok := awserrors.ResolveErrorDetail(err)
+		assert.True(t, ok, id)
+		assert.Equal(t, awserrors.ErrorInvalidAMIIDMalformed, code, id)
+		assert.Equal(t, `Invalid id: "`+id+`"`, msg, id)
+	}
+}

spinifex/domains/ec2/awsapi/image/handler_test.go

@@ -39,13 +39,6 @@ func TestCreateImage_NilNATS(t *testing.T) {
 	assert.Error(t, err)
 }
 
-func TestDescribeImages_ValidationErrors(t *testing.T) {
-	_, err := DescribeImages(context.Background(), &ec2.DescribeImagesInput{
-		ImageIds: []*string{aws.String("bad-id")},
-	}, nil, "")
-	assert.EqualError(t, err, awserrors.ErrorInvalidAMIIDMalformed)
-}
-
 func TestDescribeImages_NilNATS(t *testing.T) {
 	_, err := DescribeImages(context.Background(), nil, nil, "acct-123")
 	assert.Error(t, err)

spinifex/domains/ec2/awsapi/instance/DescribeInstances.go

@@ -89,6 +89,12 @@ func DescribeInstances(ctx context.Context, input *ec2.DescribeInstancesInput, n
 	return &ec2.DescribeInstancesOutput{Reservations: reservations}, nil
 }
 
+// ValidateDescribeInstancesInput rejects any InstanceIds entry that is not a
+// well-formed instance ID with InvalidInstanceID.Malformed.
+func ValidateDescribeInstancesInput(input *ec2.DescribeInstancesInput) error {
+	return validateInstanceIDFormat(input.InstanceIds)
+}
+
 // DescribeInstancesChecked is the customer-facing variant served by the
 // gateway's DescribeInstances action. On top of DescribeInstances' aggregation,
 // it asserts InvalidInstanceID.NotFound for any explicitly named instance ID

spinifex/domains/ec2/awsapi/instance/IamInstanceProfileAssociation.go

@@ -35,7 +35,7 @@ func resolveAndAuthorizeProfile(spec *ec2.IamInstanceProfileSpecification, iamSv
 	}
 	profile, err := iamSvc.ResolveInstanceProfile(accountID, nameOrARN)
 	if err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return nil, errors.New(awserrors.ErrorInvalidIamInstanceProfileNotFound)
 		}
 		return nil, err

spinifex/domains/ec2/awsapi/instance/IamInstanceProfileAssociation_test.go

@@ -45,7 +45,7 @@ func (f *fakeIAMService) ResolveInstanceProfile(accountID, nameOrARN string) (*i
 	if f.resolveFn != nil {
 		return f.resolveFn(accountID, nameOrARN)
 	}
-	return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+	return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", nameOrARN)
 }
 
 func (f *fakeIAMService) CanonicalResourceARN(accountID string, kind arn.IAMResourceType, name string) (string, error) {

spinifex/domains/ec2/awsapi/instance/instanceid.go

@@ -25,7 +25,7 @@ var instanceIDPattern = regexp.MustCompile(`^i-([0-9a-f]{8}|[0-9a-f]{17})$`)
 func validateInstanceIDFormat(ids []*string) error {
 	for _, id := range ids {
 		if id != nil && !instanceIDPattern.MatchString(*id) {
-			return errors.New(awserrors.ErrorInvalidInstanceIDMalformed)
+			return awserrors.Errorf(awserrors.ErrorInvalidInstanceIDMalformed, "Invalid id: %q", *id)
 		}
 	}
 	return nil

spinifex/domains/ec2/awsapi/instance/instanceid_test.go

@@ -57,8 +57,10 @@ func TestValidateInstanceIDFormat(t *testing.T) {
 	}
 	for _, id := range malformed {
 		err := validateInstanceIDFormat([]*string{aws.String(id)})
-		require.Error(t, err, id)
-		assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, err.Error(), id)
+		code, msg, ok := awserrors.ResolveErrorDetail(err)
+		require.True(t, ok, id)
+		assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, code, id)
+		assert.Equal(t, `Invalid id: "`+id+`"`, msg, id)
 	}
 	err := validateInstanceIDFormat([]*string{aws.String("i-0123456789abcdef0"), aws.String("i-abc")})
 	require.Error(t, err, "one malformed id fails the whole batch")
@@ -100,8 +102,9 @@ func TestLifecycleVerbs_MalformedIDRejectedBeforeNATS(t *testing.T) {
 	}
 	for name, call := range calls {
 		err := call()
-		require.Error(t, err, name)
-		assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, err.Error(), name)
+		code, ok := awserrors.ResolveErrorCode(err)
+		require.True(t, ok, name)
+		assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, code, name)
 	}
 	require.NoError(t, nc.Flush())
 	assert.Zero(t, requests.Load(), "no NATS request may be sent for a batch holding a malformed id")

spinifex/domains/ec2/awsapi/volume/DescribeVolumes.go

@@ -5,6 +5,7 @@ package volume
 import (
 	"context"
 	"errors"
+	"regexp"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -13,8 +14,13 @@ import (
 	"github.com/nats-io/nats.go"
 )
 
-// ValidateDescribeVolumesInput accepts a nil input and rejects any VolumeIds entry without the
-// vol- prefix with InvalidVolumeID.Malformed.
+// volumeIDSuffixPattern is the AWS volume ID shape after vol-: 8 (legacy) or
+// 17 lowercase hex characters.
+var volumeIDSuffixPattern = regexp.MustCompile(`^([0-9a-f]{8}|[0-9a-f]{17})$`)
+
+// ValidateDescribeVolumesInput accepts a nil input, rejects any VolumeIds entry without the
+// vol- prefix with InvalidVolumeID.Malformed, and any with a malformed suffix with
+// InvalidParameterValue, as AWS does.
 func ValidateDescribeVolumesInput(input *ec2.DescribeVolumesInput) error {
 	if input == nil {
 		return nil
@@ -22,9 +28,17 @@ func ValidateDescribeVolumesInput(input *ec2.DescribeVolumesInput) error {
 
 	if input.VolumeIds != nil {
 		for _, volumeId := range input.VolumeIds {
-			if volumeId != nil && !strings.HasPrefix(*volumeId, "vol-") {
+			if volumeId == nil {
+				continue
+			}
+			suffix, ok := strings.CutPrefix(*volumeId, "vol-")
+			if !ok {
 				return errors.New(awserrors.ErrorInvalidVolumeIDMalformed)
 			}
+			if !volumeIDSuffixPattern.MatchString(suffix) {
+				return awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+					"Value (%s) for parameter volumes is invalid. Expected: 'vol-...'.", *volumeId)
+			}
 		}
 	}
 
@@ -35,11 +49,6 @@ func ValidateDescribeVolumesInput(input *ec2.DescribeVolumesInput) error {
 func DescribeVolumes(ctx context.Context, input *ec2.DescribeVolumesInput, natsConn *nats.Conn, accountID string) (ec2.DescribeVolumesOutput, error) {
 	var output ec2.DescribeVolumesOutput
 
-	err := ValidateDescribeVolumesInput(input)
-	if err != nil {
-		return output, err
-	}
-
 	volumeService := ec2volume.NewNATSVolumeService(natsConn)
 	result, err := volumeService.DescribeVolumes(ctx, input, accountID)
 	if err != nil {

spinifex/domains/ec2/awsapi/volume/DescribeVolumes_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -36,9 +37,9 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
 			name: "ValidMultipleVolumeIds",
 			input: &ec2.DescribeVolumesInput{
 				VolumeIds: []*string{
-					aws.String("vol-abc123"),
-					aws.String("vol-def456"),
-					aws.String("vol-ghi789"),
+					aws.String("vol-0123456789abcdef1"),
+					aws.String("vol-1a2b3c4d"),
+					aws.String("vol-0123456789abcdef2"),
 				},
 			},
 			wantErr: false,
@@ -71,7 +72,7 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
 			name: "MixedValidAndInvalid",
 			input: &ec2.DescribeVolumesInput{
 				VolumeIds: []*string{
-					aws.String("vol-valid123"),
+					aws.String("vol-0123456789abcdef3"),
 					aws.String("invalid-id"),
 				},
 			},
@@ -81,7 +82,7 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
 		{
 			name: "NilVolumeIdInList",
 			input: &ec2.DescribeVolumesInput{
-				VolumeIds: []*string{nil, aws.String("vol-abc123")},
+				VolumeIds: []*string{nil, aws.String("vol-0123456789abcdef1")},
 			},
 			wantErr: false,
 		},
@@ -100,3 +101,15 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
 		})
 	}
 }
+
+// AWS answers a well-prefixed ID whose suffix is not 8 or 17 lowercase hex
+// characters with InvalidParameterValue, not NotFound.
+func TestValidateDescribeVolumesInput_MalformedSuffix(t *testing.T) {
+	for _, id := range []string{"vol-zzz", "vol-0123456789ABCDEF0", "vol-0123456789abcdef", "vol-"} {
+		err := ValidateDescribeVolumesInput(&ec2.DescribeVolumesInput{VolumeIds: []*string{aws.String(id)}})
+		code, msg, ok := awserrors.ResolveErrorDetail(err)
+		assert.True(t, ok, id)
+		assert.Equal(t, awserrors.ErrorInvalidParameterValue, code, id)
+		assert.Equal(t, "Value ("+id+") for parameter volumes is invalid. Expected: 'vol-...'.", msg, id)
+	}
+}

spinifex/domains/ec2/awsapi/volume/handler_test.go

@@ -96,13 +96,6 @@ func TestModifyVolume_NilNATS(t *testing.T) {
 	assert.Error(t, err)
 }
 
-func TestDescribeVolumes_ValidationErrors(t *testing.T) {
-	_, err := DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{
-		VolumeIds: []*string{aws.String("bad-id")},
-	}, nil, "")
-	assert.EqualError(t, err, awserrors.ErrorInvalidVolumeIDMalformed)
-}
-
 func TestDescribeVolumes_NilNATS(t *testing.T) {
 	_, err := DescribeVolumes(context.Background(), nil, nil, "acct-123")
 	assert.Error(t, err)

spinifex/domains/ec2/guestmetadata/metadata.go

@@ -738,7 +738,7 @@ func (s *IMDSServiceImpl) profileFor(ctx context.Context, eni *eniFacts) (*resol
 	}
 	profile, err := s.iam.ResolveInstanceProfile(ctx, eni.iamAccountID(), inst.iamInstanceProfileArn)
 	if err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return nil, roleMissProfileDeleted, nil // profile deleted; treat as no role
 		}
 		slog.ErrorContext(ctx, "IMDS: resolve instance profile failed", "account_id", eni.iamAccountID(), "arn", inst.iamInstanceProfileArn, "err", err)

spinifex/domains/ec2/guestmetadata/metadata_test.go

@@ -1152,7 +1152,7 @@ func TestHTTP_SecurityCredentialsListBackendError500(t *testing.T) {
 // genuine backend error (TestHTTP_*BackendError500) is a 500.
 func TestHTTP_IAMInfoDeletedProfile404(t *testing.T) {
 	res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/gone"}}
-	svc, _ := newTestService(res, &fakeIAM{profileErr: errors.New(awserrors.ErrorIAMNoSuchEntity)}, &fakeAssumer{})
+	svc, _ := newTestService(res, &fakeIAM{profileErr: awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile gone cannot be found.")}, &fakeAssumer{})
 	h := withTapENI(svc.httpHandler(), testENI())
 	token := issueToken(t, h)
 	rec := get(t, h, prefixMetaData+"iam/info", token)

spinifex/domains/ec2/vpc/eni.go

@@ -224,7 +224,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2.
 	groupNames := make(map[string]string)
 	s.fillSGNames(ctx, accountID, record.SecurityGroupIds, groupNames)
 	return &ec2.CreateNetworkInterfaceOutput{
-		NetworkInterface: s.eniRecordToEC2(&record, accountID, groupNames),
+		NetworkInterface: s.eniRecordToEC2(&record, accountID, groupNames, s.vpcDNSHostnames(ctx, accountID, record.VpcId)),
 	}, nil
 }
 
@@ -595,7 +595,7 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *e
 		}
 
 		s.fillSGNames(ctx, accountID, record.SecurityGroupIds, groupNames)
-		enis = append(enis, s.eniRecordToEC2(&record, accountID, groupNames))
+		enis = append(enis, s.eniRecordToEC2(&record, accountID, groupNames, s.vpcDNSHostnames(ctx, accountID, record.VpcId)))
 	}
 
 	// If specific ENI IDs were requested but not found, return error
@@ -1076,9 +1076,26 @@ func (s *VPCServiceImpl) fillSGNames(ctx context.Context, accountID string, ids
 	}
 }
 
+// vpcDNSHostnames reports whether the VPC has enableDnsHostnames on. An
+// unreadable VPC reads as off, so its interfaces go unnamed.
+func (s *VPCServiceImpl) vpcDNSHostnames(ctx context.Context, accountID, vpcID string) bool {
+	entry, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID))
+	if err != nil {
+		slog.WarnContext(ctx, "vpcDNSHostnames: VPC read failed", "vpcId", vpcID, "err", err)
+		return false
+	}
+	var rec VPCRecord
+	if err := json.Unmarshal(entry.Value(), &rec); err != nil {
+		slog.WarnContext(ctx, "vpcDNSHostnames: VPC unmarshal failed", "vpcId", vpcID, "err", err)
+		return false
+	}
+	return rec.EnableDnsHostnames
+}
+
 // eniRecordToEC2 converts an ENI record to an EC2 NetworkInterface, naming
-// each security group from groupNames.
-func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, groupNames map[string]string) *ec2.NetworkInterface {
+// each security group from groupNames. AWS names the interface only while its
+// VPC has enableDnsHostnames on.
+func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, groupNames map[string]string, dnsHostnames bool) *ec2.NetworkInterface {
 	// ENIs with spinifex:managed-by tag are system-managed (e.g. by ELBv2)
 	requesterManaged := record.Tags["spinifex:managed-by"] != ""
 
@@ -1116,7 +1133,7 @@ func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, gro
 		eni.Groups = groups
 	}
 
-	if s.privateDNSName != nil && record.PrivateIpAddress != "" {
+	if dnsHostnames && s.privateDNSName != nil && record.PrivateIpAddress != "" {
 		if name := s.privateDNSName(record.PrivateIpAddress); name != "" {
 			eni.PrivateDnsName = aws.String(name)
 			eni.PrivateIpAddresses[0].PrivateDnsName = aws.String(name)

spinifex/domains/ec2/vpc/eni_test.go

@@ -55,15 +55,11 @@ func TestNetworkInterface_NamesGroupsAndPrivateAddress(t *testing.T) {
 	vpcId := createTestVPC(t, svc, "10.0.0.0/16")
 	subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
 	sgId := createTestSG(t, svc, vpcId, "web")
+	setHostnames(t, svc, vpcId, true)
 
 	// Without a namer an interface carries no private name.
 	unnamed := createTestENI(t, svc, subnetId)
-	desc, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
-		NetworkInterfaceIds: []*string{aws.String(unnamed)},
-	}, testAccountID)
-	require.NoError(t, err)
-	require.Len(t, desc.NetworkInterfaces, 1)
-	assert.Nil(t, desc.NetworkInterfaces[0].PrivateDnsName)
+	assert.Nil(t, describeTestENI(t, svc, unnamed).PrivateDnsName)
 
 	svc.SetPrivateDNSNamer(func(ip string) string { return "name-of-" + ip })
 	out, err := svc.CreateNetworkInterface(context.Background(), &ec2.CreateNetworkInterfaceInput{
@@ -72,13 +68,8 @@ func TestNetworkInterface_NamesGroupsAndPrivateAddress(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 
-	desc, err = svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
-		NetworkInterfaceIds: []*string{out.NetworkInterface.NetworkInterfaceId},
-	}, testAccountID)
-	require.NoError(t, err)
-	require.Len(t, desc.NetworkInterfaces, 1)
-
-	for name, eni := range map[string]*ec2.NetworkInterface{"create": out.NetworkInterface, "describe": desc.NetworkInterfaces[0]} {
+	desc := describeTestENI(t, svc, *out.NetworkInterface.NetworkInterfaceId)
+	for name, eni := range map[string]*ec2.NetworkInterface{"create": out.NetworkInterface, "describe": desc} {
 		want := "name-of-" + aws.StringValue(eni.PrivateIpAddress)
 		assert.Equal(t, want, aws.StringValue(eni.PrivateDnsName), name)
 		require.Len(t, eni.PrivateIpAddresses, 1, name)
@@ -89,6 +80,58 @@ func TestNetworkInterface_NamesGroupsAndPrivateAddress(t *testing.T) {
 	}
 }
 
+func TestNetworkInterface_PrivateDnsNameFollowsVpcDnsHostnames(t *testing.T) {
+	t.Parallel()
+	svc := setupTestVPCService(t)
+	svc.SetPrivateDNSNamer(func(ip string) string { return "name-of-" + ip })
+	vpcId := createTestVPC(t, svc, "10.0.0.0/16")
+	subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
+
+	// A new VPC has enableDnsHostnames off, so its interfaces go unnamed.
+	out, err := svc.CreateNetworkInterface(context.Background(), &ec2.CreateNetworkInterfaceInput{
+		SubnetId: aws.String(subnetId),
+	}, testAccountID)
+	require.NoError(t, err)
+	eniId := *out.NetworkInterface.NetworkInterfaceId
+	for name, eni := range map[string]*ec2.NetworkInterface{"create": out.NetworkInterface, "describe": describeTestENI(t, svc, eniId)} {
+		assert.Nil(t, eni.PrivateDnsName, name)
+		require.Len(t, eni.PrivateIpAddresses, 1, name)
+		assert.Nil(t, eni.PrivateIpAddresses[0].PrivateDnsName, name)
+	}
+
+	// Turning the flag on names the existing interface.
+	setHostnames(t, svc, vpcId, true)
+	eni := describeTestENI(t, svc, eniId)
+	want := "name-of-" + aws.StringValue(eni.PrivateIpAddress)
+	assert.Equal(t, want, aws.StringValue(eni.PrivateDnsName))
+	assert.Equal(t, want, aws.StringValue(eni.PrivateIpAddresses[0].PrivateDnsName))
+
+	// And turning it off again drops the name.
+	setHostnames(t, svc, vpcId, false)
+	eni = describeTestENI(t, svc, eniId)
+	assert.Nil(t, eni.PrivateDnsName)
+	assert.Nil(t, eni.PrivateIpAddresses[0].PrivateDnsName)
+}
+
+func setHostnames(t *testing.T, svc *VPCServiceImpl, vpcId string, on bool) {
+	t.Helper()
+	_, err := svc.ModifyVpcAttribute(context.Background(), &ec2.ModifyVpcAttributeInput{
+		VpcId:              aws.String(vpcId),
+		EnableDnsHostnames: &ec2.AttributeBooleanValue{Value: aws.Bool(on)},
+	}, testAccountID)
+	require.NoError(t, err)
+}
+
+func describeTestENI(t *testing.T, svc *VPCServiceImpl, eniId string) *ec2.NetworkInterface {
+	t.Helper()
+	desc, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
+		NetworkInterfaceIds: []*string{aws.String(eniId)},
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, desc.NetworkInterfaces, 1)
+	return desc.NetworkInterfaces[0]
+}
+
 func TestCreateNetworkInterface_SequentialIPs(t *testing.T) {
 	t.Parallel()
 	svc := setupTestVPCService(t)

spinifex/domains/ecs/records.go

@@ -272,11 +272,15 @@ type TaskRecord struct {
 	// Group / StartedBy mirror the AWS task fields. A service's tasks carry
 	// Group="service:{name}" so the reconciler counts them and the task-state
 	// hook resolves a RUNNING/STOPPED task back to its owning service.
-	Group                string `json:"group,omitempty"`
-	StartedBy            string `json:"startedBy,omitempty"`
-	TaskDefFamily        string `json:"taskDefFamily"`
-	TaskDefRevision      int    `json:"taskDefRevision"`
-	TaskDefARN           string `json:"taskDefArn"`
+	Group           string `json:"group,omitempty"`
+	StartedBy       string `json:"startedBy,omitempty"`
+	TaskDefFamily   string `json:"taskDefFamily"`
+	TaskDefRevision int    `json:"taskDefRevision"`
+	TaskDefARN      string `json:"taskDefArn"`
+	// CredID is the secret under which the task's containers fetch task-role
+	// credentials from the agent. Set only when the task has a role; it must
+	// never be projected into an API response or logged.
+	CredID               string `json:"credId,omitempty"`
 	ContainerInstanceID  string `json:"containerInstanceId,omitempty"`
 	ContainerInstanceARN string `json:"containerInstanceArn,omitempty"`
 	DesiredStatus        string `json:"desiredStatus"`

spinifex/domains/ecs/service.go

@@ -288,7 +288,7 @@ func (s *Service) DescribeClusters(ctx context.Context, input *ecs.DescribeClust
 		names = []string{defaultCluster}
 	}
 	includeSettings := slices.Contains(awsStringSlice(input.Include), ecs.ClusterFieldSettings)
-	out := &ecs.DescribeClustersOutput{}
+	out := &ecs.DescribeClustersOutput{Clusters: []*ecs.Cluster{}, Failures: []*ecs.Failure{}}
 	for _, name := range names {
 		name = ClusterShortName(name)
 		var rec ClusterRecord
@@ -458,7 +458,7 @@ func (s *Service) RegisterTaskDefinition(ctx context.Context, input *ecs.Registe
 	if family == "" {
 		return nil, errors.New(awserrors.ErrorECSInvalidParameter)
 	}
-	if err := validateContainerDefs(input.ContainerDefinitions); err != nil {
+	if err := validateContainerDefs(input.ContainerDefinitions, aws.StringValue(input.Memory)); err != nil {
 		return nil, err
 	}
 	warnUnsupportedLogDrivers(ctx, family, input.ContainerDefinitions)
@@ -497,12 +497,23 @@ func (s *Service) RegisterTaskDefinition(ctx context.Context, input *ecs.Registe
 // The test here is per value, not per field: an empty list or a false/unset
 // flag requests nothing and is accepted, because there is nothing to enforce
 // or refuse. Only a value that asks for the restriction is rejected.
-func validateContainerDefs(defs []*ecs.ContainerDefinition) error {
+//
+// Ahead of those, AWS's own rules: every container needs an image, and without
+// a task-level memory every container needs memory or memoryReservation.
+func validateContainerDefs(defs []*ecs.ContainerDefinition, taskMemory string) error {
 	for _, c := range defs {
 		if c == nil {
 			continue
 		}
 		name := aws.StringValue(c.Name)
+		if aws.StringValue(c.Image) == "" {
+			return awserrors.Errorf(awserrors.ErrorECSClientException,
+				"Container.image should not be null or empty.")
+		}
+		if taskMemory == "" && c.Memory == nil && c.MemoryReservation == nil {
+			return awserrors.Errorf(awserrors.ErrorECSClientException,
+				"Invalid setting for container '%s'. At least one of 'memory' or 'memoryReservation' must be specified.", name)
+		}
 		if len(c.Secrets) > 0 {
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}

spinifex/domains/ecs/service_test.go

@@ -2,7 +2,9 @@ package ecs
 
 import (
 	"context"
+	"encoding/json"
 	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
+	"maps"
 	"strconv"
 	"testing"
 	"time"
@@ -302,6 +304,89 @@ func TestService_RunTask_AssignCarriesTaskRole(t *testing.T) {
 	assert.Equal(t, roleARN, poll.Assignments[0].TaskRoleARN)
 }
 
+// A role task's credential ID is a scheduler-minted secret: not the public task
+// ID, distinct per task, stable on re-delivery, persisted on the task record and
+// absent from every API response. A role-less task gets none.
+func TestService_TaskCredIDIsMintedSecret(t *testing.T) {
+	ctx := context.Background()
+	svc, _ := newTestService(t)
+	_, err := svc.CreateCluster(ctx, &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
+	require.NoError(t, err)
+	_, err = svc.RegisterTaskDefinition(ctx, &ecs.RegisterTaskDefinitionInput{
+		Family:      aws.String("victim"),
+		TaskRoleArn: aws.String("arn:aws:iam::123456789012:role/task-high"),
+		ContainerDefinitions: []*ecs.ContainerDefinition{{
+			Name: aws.String("app"), Image: aws.String("registry/app:1"),
+			Cpu: aws.Int64(128), Memory: aws.Int64(256), Essential: aws.Bool(true),
+		}},
+	}, testAccountID)
+	require.NoError(t, err)
+	registerTaskDef(t, svc, "attacker", 128, 256)
+	registerInstance(t, svc, "web", "i-1", 1024, 2048)
+
+	run, err := svc.RunTask(ctx, &ecs.RunTaskInput{Cluster: aws.String("web"), TaskDefinition: aws.String("victim")}, testAccountID)
+	require.NoError(t, err)
+	start, err := svc.StartTask(ctx, &ecs.StartTaskInput{
+		Cluster: aws.String("web"), TaskDefinition: aws.String("victim"), ContainerInstances: []*string{aws.String("i-1")},
+	}, testAccountID)
+	require.NoError(t, err)
+	_, err = svc.StartTask(ctx, &ecs.StartTaskInput{
+		Cluster: aws.String("web"), TaskDefinition: aws.String("attacker"), ContainerInstances: []*string{aws.String("i-1")},
+	}, testAccountID)
+	require.NoError(t, err)
+
+	poll := func() map[string]ecsv1.Assign {
+		p, perr := svc.PollAssignments(ctx, &PollAssignmentsInput{Cluster: "web", ContainerInstance: "i-1"}, testAccountID)
+		require.NoError(t, perr)
+		byTask := map[string]ecsv1.Assign{}
+		for _, as := range p.Assignments {
+			byTask[as.TaskID] = as
+		}
+		return byTask
+	}
+	first, second := poll(), poll()
+	require.Len(t, first, 3)
+
+	kv, err := svc.bucket(ctx, testAccountID)
+	require.NoError(t, err)
+	var apiJSON []byte
+	for _, out := range []any{run, start} {
+		b, merr := json.Marshal(out)
+		require.NoError(t, merr)
+		apiJSON = append(apiJSON, b...)
+	}
+	var taskARNs []*string
+	for as := range maps.Values(first) {
+		taskARNs = append(taskARNs, aws.String(as.TaskARN))
+	}
+	desc, err := svc.DescribeTasks(ctx, &ecs.DescribeTasksInput{Cluster: aws.String("web"), Tasks: taskARNs}, testAccountID)
+	require.NoError(t, err)
+	b, err := json.Marshal(desc)
+	require.NoError(t, err)
+	apiJSON = append(apiJSON, b...)
+
+	seen := map[string]bool{}
+	for taskID, as := range first {
+		if as.TaskRoleARN == "" {
+			assert.Empty(t, as.CredID, "role-less task got a credential ID")
+			continue
+		}
+		require.NotEmpty(t, as.CredID)
+		assert.NotEqual(t, taskID, as.CredID)
+		assert.False(t, seen[as.CredID], "credential ID reused across tasks")
+		seen[as.CredID] = true
+		assert.Equal(t, as.CredID, second[taskID].CredID, "re-delivered assign changed credential ID")
+
+		var rec TaskRecord
+		found, gerr := getJSON(ctx, kv, TaskKey("web", taskID), &rec)
+		require.NoError(t, gerr)
+		require.True(t, found)
+		assert.Equal(t, as.CredID, rec.CredID)
+		assert.NotContains(t, string(apiJSON), as.CredID, "credential ID leaked in an API response")
+	}
+	assert.Len(t, seen, 2)
+}
+
 // PollAssignments is at-least-once: re-poll without ack re-delivers the assign;
 // acking it (then STOPPED) drains the inbox so it is never re-delivered.
 func TestService_PollAssignments_AckAndReclaim(t *testing.T) {
@@ -812,6 +897,12 @@ func TestService_ListTasks_EmptyIsPresentNotAbsent(t *testing.T) {
 	assert.Empty(t, out.TaskArns)
 }
 
+func TestService_ListTasks_ClusterNotFound(t *testing.T) {
+	svc, _ := newTestService(t)
+	_, err := svc.ListTasks(context.Background(), &ecs.ListTasksInput{Cluster: aws.String("ghost")}, testAccountID)
+	require.EqualError(t, err, awserrors.ErrorECSClusterNotFound)
+}
+
 func TestService_ListContainerInstances_EmptyIsPresentNotAbsent(t *testing.T) {
 	svc, _ := newTestService(t)
 	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
@@ -829,3 +920,21 @@ func TestAccountIDFromBucket(t *testing.T) {
 	_, ok = accountIDFromBucket("some-other-bucket")
 	assert.False(t, ok)
 }
+
+func TestService_DescribeClusters_ListsAlwaysPresent(t *testing.T) {
+	svc, _ := newTestService(t)
+	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
+	require.NoError(t, err)
+
+	found, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{Clusters: []*string{aws.String("web")}}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, found.Clusters, 1)
+	assert.NotNil(t, found.Failures)
+	assert.Empty(t, found.Failures)
+
+	miss, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{Clusters: []*string{aws.String("nope")}}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, miss.Failures, 1)
+	assert.NotNil(t, miss.Clusters)
+	assert.Empty(t, miss.Clusters)
+}

spinifex/domains/ecs/services_test.go

@@ -313,7 +313,7 @@ func TestService_CreateService_ProjectsNetworkConfigurationAndManagedTags(t *tes
 		Family:      aws.String("app"),
 		NetworkMode: aws.String(NetworkModeAwsvpc),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -352,7 +352,7 @@ func TestService_UpdateService_ChangesNetworkConfiguration(t *testing.T) {
 		Family:      aws.String("app"),
 		NetworkMode: aws.String(NetworkModeAwsvpc),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)

spinifex/domains/ecs/task_accounting_test.go

@@ -16,6 +16,8 @@ import (
 // agent to reach anything other than PENDING.
 func seedTasks(t *testing.T, svc *ecsdomain.Service, cluster string, recs ...*ecsdomain.TaskRecord) {
 	t.Helper()
+	_, err := svc.CreateCluster(t.Context(), &ecs.CreateClusterInput{ClusterName: aws.String(cluster)}, ecsdomain.TestAccountID)
+	require.NoError(t, err)
 	kv, err := svc.Bucket(t.Context(), ecsdomain.TestAccountID)
 	require.NoError(t, err)
 	for _, r := range recs {

spinifex/domains/ecs/taskdef_validation_test.go

@@ -19,7 +19,7 @@ func TestRegisterTaskDefinition_RejectsSecrets(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			Secrets: []*ecs.Secret{{
 				Name: aws.String("DB_PASSWORD"), ValueFrom: aws.String("arn:aws:ssm:::parameter/db"),
 			}},
@@ -37,7 +37,7 @@ func TestRegisterTaskDefinition_AcceptsUnsupportedLogDriver(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LogConfiguration: &ecs.LogConfiguration{
 				LogDriver: aws.String("awslogs"),
 				Options:   map[string]*string{"awslogs-group": aws.String("/ecs/app")},
@@ -67,7 +67,7 @@ func TestRegisterTaskDefinition_EchoesRequiresCompatibilities(t *testing.T) {
 		Family:                  aws.String("app"),
 		RequiresCompatibilities: aws.StringSlice([]string{"EC2"}),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -91,7 +91,7 @@ func TestRegisterTaskDefinition_EchoesRuntimePlatform(t *testing.T) {
 			OperatingSystemFamily: aws.String("LINUX"),
 		},
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -113,7 +113,7 @@ func TestRegisterTaskDefinition_RuntimePlatformOmittedWhenUnset(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -253,7 +253,7 @@ func TestRegisterTaskDefinition_EchoesRuntimeFields(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			User:                   aws.String("1000:1000"),
 			ReadonlyRootFilesystem: aws.Bool(true),
 			Privileged:             aws.Bool(false),
@@ -310,7 +310,7 @@ func TestRegisterTaskDefinition_RuntimeFieldsOmittedWhenUnset(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -347,7 +347,7 @@ func TestRegisterTaskDefinition_EchoesEmptyCollections(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			Environment:     []*ecs.KeyValuePair{},
 			MountPoints:     []*ecs.MountPoint{},
 			VolumesFrom:     []*ecs.VolumeFrom{},
@@ -386,7 +386,7 @@ func TestRegisterTaskDefinition_FillsAWSDefaults(t *testing.T) {
 		Family:      aws.String("app"),
 		NetworkMode: aws.String("awsvpc"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			PortMappings: []*ecs.PortMapping{{ContainerPort: aws.Int64(80)}},
 		}},
 	}, testAccountID)
@@ -420,7 +420,7 @@ func TestRegisterTaskDefinition_BridgeLeavesHostPortUnset(t *testing.T) {
 		Family:      aws.String("app"),
 		NetworkMode: aws.String("bridge"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			PortMappings: []*ecs.PortMapping{{ContainerPort: aws.Int64(53), Protocol: aws.String("udp")}},
 		}},
 	}, testAccountID)
@@ -435,7 +435,7 @@ func TestRegisterTaskDefinition_AcceptsEmptyRuntimeFields(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			MountPoints:    []*ecs.MountPoint{},
 			VolumesFrom:    []*ecs.VolumeFrom{},
 			SystemControls: []*ecs.SystemControl{},
@@ -455,7 +455,7 @@ func TestRegisterTaskDefinition_RejectsMountPoints(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			MountPoints: []*ecs.MountPoint{{
 				SourceVolume: aws.String("data"), ContainerPath: aws.String("/data"),
 			}},
@@ -476,7 +476,7 @@ func TestRegisterTaskDefinition_RejectsVolumesFrom(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			VolumesFrom: []*ecs.VolumeFrom{{SourceContainer: aws.String("other")}},
 		}},
 	}, testAccountID)
@@ -494,7 +494,7 @@ func TestRegisterTaskDefinition_RejectsLinuxParametersDevices(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LinuxParameters: &ecs.LinuxParameters{
 				Devices: []*ecs.Device{{HostPath: aws.String("/dev/fuse")}},
 			},
@@ -514,7 +514,7 @@ func TestRegisterTaskDefinition_RejectsInitProcessEnabled(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LinuxParameters: &ecs.LinuxParameters{InitProcessEnabled: aws.Bool(true)},
 		}},
 	}, testAccountID)
@@ -533,7 +533,7 @@ func TestRegisterTaskDefinition_RejectsSharedMemoryAndTmpfs(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("shm"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LinuxParameters: &ecs.LinuxParameters{SharedMemorySize: aws.Int64(64)},
 		}},
 	}, testAccountID)
@@ -545,7 +545,7 @@ func TestRegisterTaskDefinition_RejectsSharedMemoryAndTmpfs(t *testing.T) {
 	_, err = svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("tmpfs"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LinuxParameters: &ecs.LinuxParameters{
 				Tmpfs: []*ecs.Tmpfs{{ContainerPath: aws.String("/tmp"), Size: aws.Int64(64)}},
 			},
@@ -567,7 +567,7 @@ func TestRunTask_AssignCarriesRuntimeFields(t *testing.T) {
 	_, err = svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			User:                   aws.String("1000"),
 			ReadonlyRootFilesystem: aws.Bool(true),
 			Privileged:             aws.Bool(true),
@@ -608,3 +608,42 @@ func TestRunTask_AssignCarriesRuntimeFields(t *testing.T) {
 	assert.Equal(t, "net.core.somaxconn", ac.SystemControls[0].Namespace)
 	assert.Equal(t, []string{"SYS_PTRACE"}, ac.CapAdd)
 }
+
+// AWS rejects a container with no image before checking its memory, and
+// without task-level memory requires memory or memoryReservation per container.
+func TestRegisterTaskDefinition_RequiresImageAndMemory(t *testing.T) {
+	tests := []struct {
+		name       string
+		taskMemory *string
+		container  *ecs.ContainerDefinition
+		wantErr    string
+	}{
+		{"no image or memory", nil, &ecs.ContainerDefinition{Name: aws.String("web")},
+			"Container.image should not be null or empty."},
+		{"empty image", aws.String("512"), &ecs.ContainerDefinition{Name: aws.String("web"), Image: aws.String("")},
+			"Container.image should not be null or empty."},
+		{"no memory", nil, &ecs.ContainerDefinition{Name: aws.String("web"), Image: aws.String("registry/web:1")},
+			"Invalid setting for container 'web'. At least one of 'memory' or 'memoryReservation' must be specified."},
+		{"memoryReservation only", nil, &ecs.ContainerDefinition{
+			Name: aws.String("web"), Image: aws.String("registry/web:1"), MemoryReservation: aws.Int64(128)}, ""},
+		{"task-level memory", aws.String("512"), &ecs.ContainerDefinition{
+			Name: aws.String("web"), Image: aws.String("registry/web:1")}, ""},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			svc, _ := newTestService(t)
+			_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
+				Family: aws.String("app"), Memory: tc.taskMemory,
+				ContainerDefinitions: []*ecs.ContainerDefinition{tc.container},
+			}, testAccountID)
+			if tc.wantErr == "" {
+				require.NoError(t, err)
+				return
+			}
+			code, msg, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok)
+			assert.Equal(t, awserrors.ErrorECSClientException, code)
+			assert.Equal(t, tc.wantErr, msg)
+		})
+	}
+}

spinifex/domains/ecs/tasks.go

@@ -237,6 +237,9 @@ func (s *Service) newTaskRecord(accountID, cluster, taskID string, td *taskdefin
 		GPU:                  gpu,
 		CreatedAt:            now,
 	}
+	if td.TaskRoleArn != "" {
+		rec.CredID = uuid.NewV4().String()
+	}
 	for _, c := range td.Containers {
 		rec.Containers = append(rec.Containers, ContainerState{Name: c.Name, Status: TaskStatusPending})
 	}
@@ -257,6 +260,7 @@ func (s *Service) publishAssign(ctx context.Context, kv jetstream.KeyValue, acco
 		TaskDefFamily:    td.Family,
 		TaskDefRevision:  td.Revision,
 		TaskRoleARN:      td.TaskRoleArn,
+		CredID:           rec.CredID,
 		ExecutionRoleARN: td.ExecutionRoleArn,
 		ENIID:            rec.ENIID,
 		ENIMacAddress:    rec.ENIMacAddress,
@@ -303,6 +307,14 @@ func (s *Service) ListTasks(ctx context.Context, input *ecs.ListTasksInput, acco
 	if err != nil {
 		return nil, err
 	}
+	var clusterRec ClusterRecord
+	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
+	if err != nil {
+		return nil, err
+	}
+	if !found {
+		return nil, errors.New(awserrors.ErrorECSClusterNotFound)
+	}
 	keys, err := keysWithPrefix(ctx, kv, TasksPrefix(cluster))
 	if err != nil {
 		return nil, err

spinifex/domains/eks/addon/catalog.go

@@ -4,8 +4,14 @@ import (
 	"fmt"
 	"maps"
 	"slices"
+	"strconv"
+	"strings"
 )
 
+// firstBuild is the EKS build number of every bundled version; it moves per
+// version when Spinifex rebuilds a bundle.
+const firstBuild = 1
+
 // NvidiaDevicePlugin is staged by a GPU node group's launch (EnsureGPUDevicePlugin),
 // never user-requested directly.
 const NvidiaDevicePlugin = "nvidia-device-plugin"
@@ -18,6 +24,11 @@ type Spec struct {
 	// Versions lists supported versions newest-first; [0] is the default.
 	Versions       []string
 	DefaultVersion string
+	// Builds maps each upstream version to its EKS build number, the n in
+	// v<upstream>-eksbuild.<n>.
+	Builds map[string]int
+	// Type is the AWS add-on type reported by DescribeAddonVersions.
+	Type string
 	// RequiresIRSA indicates the add-on needs an IAM role for service-account binding.
 	RequiresIRSA bool
 	Description  string
@@ -28,26 +39,26 @@ type Spec struct {
 
 // catalog is the bundled add-on registry. Keep newest version first per slice.
 var catalog = buildCatalog(
-	newSpec("aws-load-balancer-controller", true,
+	typed("networking", newSpec("aws-load-balancer-controller", true,
 		"Provisions ELBv2 load balancers for Kubernetes Service/Ingress resources.",
-		"2.11.0"),
-	newSpec("argocd", false,
+		"2.11.0")),
+	typed("gitops", newSpec("argocd", false,
 		"Declarative GitOps continuous delivery for Kubernetes.",
-		"3.0.23"),
-	newSpec("aws-ebs-csi-driver", true,
+		"3.0.23")),
+	typed("storage", newSpec("aws-ebs-csi-driver", true,
 		"Container Storage Interface driver for Amazon EBS (Viperblock) volumes.",
-		"1.40.1"),
+		"1.40.1")),
 	// Staged automatically for GPU node groups, so hidden from the public
 	// catalogue like spinifex-noop but still creatable by name.
-	hidden(newSpec(NvidiaDevicePlugin, false,
+	hidden(typed("accelerator", newSpec(NvidiaDevicePlugin, false,
 		"NVIDIA device plugin exposing nvidia.com/gpu allocatable via CDI on GPU-tainted nodes.",
-		"0.17.4")),
+		"0.17.4"))),
 	// spinifex-noop is the delivery-transport fixture: a trivial bundle
 	// (Namespace + ConfigMap) the add-on e2e uses to prove stage, render,
 	// auto-deploy, ACTIVE and delete round-trip without a real workload.
-	hidden(newSpec("spinifex-noop", false,
+	hidden(typed("fixture", newSpec("spinifex-noop", false,
 		"No-op delivery-transport fixture (Namespace + ConfigMap).",
-		"0.1.0")),
+		"0.1.0"))),
 )
 
 // hidden marks a spec as internal: still creatable, but absent from the
@@ -57,6 +68,12 @@ func hidden(s Spec) Spec {
 	return s
 }
 
+// typed sets the AWS add-on type reported by DescribeAddonVersions.
+func typed(addonType string, s Spec) Spec {
+	s.Type = addonType
+	return s
+}
+
 // newSpec builds a spec with the first version as its default.
 func newSpec(name string, requiresIRSA bool, description string, versions ...string) Spec {
 	spec := Spec{
@@ -68,6 +85,10 @@ func newSpec(name string, requiresIRSA bool, description string, versions ...str
 	if len(versions) > 0 {
 		spec.DefaultVersion = versions[0]
 	}
+	spec.Builds = make(map[string]int, len(versions))
+	for _, v := range versions {
+		spec.Builds[v] = firstBuild
+	}
 	return spec
 }
 
@@ -111,6 +132,50 @@ func (s Spec) SupportsVersion(version string) bool {
 	return slices.Contains(s.Versions, version)
 }
 
+// awsForm renders a catalog version as v<upstream>-eksbuild.<build>.
+func (s Spec) awsForm(upstream string) string {
+	build, ok := s.Builds[upstream]
+	if !ok {
+		build = firstBuild
+	}
+	return "v" + upstream + "-eksbuild." + strconv.Itoa(build)
+}
+
+// AWSVersion returns the AWS form of a stored version. A version the catalogue
+// does not list, or one already in AWS form, is returned unchanged.
+func (s Spec) AWSVersion(stored string) string {
+	if !slices.Contains(s.Versions, stored) {
+		return stored
+	}
+	return s.awsForm(stored)
+}
+
+// AWSVersions returns every supported version in AWS form, newest first.
+func (s Spec) AWSVersions() []string {
+	out := make([]string, len(s.Versions))
+	for i, v := range s.Versions {
+		out[i] = s.awsForm(v)
+	}
+	return out
+}
+
+// ParseVersion maps a client-supplied version, in AWS form or bare upstream
+// form, to the upstream version stored on the record. Nothing else is accepted.
+func (s Spec) ParseVersion(input string) (string, bool) {
+	if slices.Contains(s.Versions, input) {
+		return input, true
+	}
+	if !strings.HasPrefix(input, "v") {
+		return "", false
+	}
+	for _, v := range s.Versions {
+		if input == s.awsForm(v) {
+			return v, true
+		}
+	}
+	return "", false
+}
+
 // Specs returns every catalogue entry, hidden ones included, sorted by name.
 func Specs() []Spec {
 	names := slices.Sorted(maps.Keys(catalog))

spinifex/domains/eks/addon/catalog_test.go

@@ -100,3 +100,78 @@ func TestValidateCatalog_ChecksTheBundledCatalog(t *testing.T) {
 	t.Cleanup(func() { catalog = previous })
 	require.EqualError(t, ValidateCatalog(), `add-on "broken" has no versions`)
 }
+
+// buildSpec is a two-version spec whose newest version is on EKS build 2, so
+// a test cannot pass by hardcoding build 1.
+func buildSpec() Spec {
+	return Spec{
+		Name:     "fixture",
+		Versions: []string{"2.0.0", "1.0.0"},
+		Builds:   map[string]int{"2.0.0": 2, "1.0.0": 1},
+	}
+}
+
+func TestSpec_AWSVersionUsesTheCatalogBuildNumber(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	assert.Equal(t, "v2.0.0-eksbuild.2", spec.AWSVersion("2.0.0"))
+	assert.Equal(t, "v1.0.0-eksbuild.1", spec.AWSVersion("1.0.0"))
+}
+
+func TestSpec_AWSVersionIsIdempotent(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	assert.Equal(t, "v2.0.0-eksbuild.2", spec.AWSVersion("v2.0.0-eksbuild.2"),
+		"a record already holding the AWS form must not be wrapped twice")
+}
+
+func TestSpec_AWSVersionsAreNewestFirst(t *testing.T) {
+	t.Parallel()
+	assert.Equal(t, []string{"v2.0.0-eksbuild.2", "v1.0.0-eksbuild.1"}, buildSpec().AWSVersions())
+}
+
+func TestSpec_ParseVersionAcceptsOnlyTheExactForms(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	for input, want := range map[string]string{
+		"v2.0.0-eksbuild.2": "2.0.0",
+		"2.0.0":             "2.0.0",
+		"v1.0.0-eksbuild.1": "1.0.0",
+		"1.0.0":             "1.0.0",
+	} {
+		got, ok := spec.ParseVersion(input)
+		assert.True(t, ok, "%q must be accepted", input)
+		assert.Equal(t, want, got, "%q", input)
+	}
+}
+
+func TestSpec_ParseVersionRefusesEverythingElse(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	for _, input := range []string{
+		"v2.0.0-eksbuild.1", // wrong build for 2.0.0
+		"v2.0.0-eksbuild.3",
+		"v1.0.0-eksbuild.2", // wrong build for 1.0.0
+		"v2.0.0",            // missing build
+		"2.0",               // partial
+		"v2.0.0-eksbuild.",
+		"2.0.0-eksbuild.2", // missing v
+		"V2.0.0-eksbuild.2",
+		" 2.0.0",
+		"9.9.9",
+		"",
+	} {
+		_, ok := spec.ParseVersion(input)
+		assert.False(t, ok, "%q must be refused", input)
+	}
+}
+
+func TestCatalog_EveryVersionHasAnExplicitBuild(t *testing.T) {
+	t.Parallel()
+	for _, spec := range Specs() {
+		for _, v := range spec.Versions {
+			assert.Equal(t, 1, spec.Builds[v], "%s %s needs an explicit build number", spec.Name, v)
+			assert.Equal(t, "v"+v+"-eksbuild.1", spec.AWSVersion(v), "%s %s", spec.Name, v)
+		}
+	}
+}

spinifex/domains/eks/awsapi/addon_versions_query_test.go

@@ -0,0 +1,105 @@
+package awsapi
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// captureDescribeAddonVersions answers eks.DescribeAddonVersions and returns a
+// function reading the last input the gateway sent to the service.
+func captureDescribeAddonVersions(t *testing.T, nc *nats.Conn) func() eks.DescribeAddonVersionsInput {
+	t.Helper()
+	var (
+		mu   sync.Mutex
+		last eks.DescribeAddonVersionsInput
+	)
+	sub, err := nc.Subscribe("eks.DescribeAddonVersions", func(m *nats.Msg) {
+		mu.Lock()
+		last = eks.DescribeAddonVersionsInput{}
+		_ = json.Unmarshal(m.Data, &last)
+		mu.Unlock()
+		_ = m.Respond([]byte(`{}`))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	return func() eks.DescribeAddonVersionsInput {
+		mu.Lock()
+		defer mu.Unlock()
+		return last
+	}
+}
+
+// The gateway folds a GET query string into the body as a JSON object of
+// string lists; the model binds every field below from the query.
+func TestDescribeAddonVersions_PassesEveryModelQueryParameter(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	got := captureDescribeAddonVersions(t, nc)
+
+	query := []byte(`{"addonName":["aws-ebs-csi-driver"],"kubernetesVersion":["1.32"],` +
+		`"maxResults":["2"],"nextToken":["tok"],"types":["storage","networking"],` +
+		`"publishers":["eks"],"owners":["aws"]}`)
+	_, err := DescribeAddonVersions(context.Background(), nc, acct, query)
+	require.NoError(t, err)
+
+	in := got()
+	assert.Equal(t, "aws-ebs-csi-driver", aws.StringValue(in.AddonName))
+	assert.Equal(t, "1.32", aws.StringValue(in.KubernetesVersion))
+	assert.Equal(t, int64(2), aws.Int64Value(in.MaxResults))
+	assert.Equal(t, "tok", aws.StringValue(in.NextToken))
+	assert.Equal(t, []string{"storage", "networking"}, aws.StringValueSlice(in.Types))
+	assert.Equal(t, []string{"eks"}, aws.StringValueSlice(in.Publishers))
+	assert.Equal(t, []string{"aws"}, aws.StringValueSlice(in.Owners))
+}
+
+func TestDescribeAddonVersions_NoQueryLeavesEveryFilterUnset(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	got := captureDescribeAddonVersions(t, nc)
+
+	_, err := DescribeAddonVersions(context.Background(), nc, acct, nil)
+	require.NoError(t, err)
+
+	in := got()
+	assert.Nil(t, in.AddonName)
+	assert.Nil(t, in.KubernetesVersion)
+	assert.Nil(t, in.MaxResults)
+	assert.Nil(t, in.NextToken)
+	assert.Empty(t, in.Types)
+}
+
+func TestDescribeAddonVersions_RefusesMalformedQuery(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	captureDescribeAddonVersions(t, nc)
+
+	_, err := DescribeAddonVersions(context.Background(), nc, acct, []byte(`{"maxResults":["many"]}`))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "maxResults")
+
+	_, err = DescribeAddonVersions(context.Background(), nc, acct, []byte(`not json`))
+	require.Error(t, err)
+}
+
+// The route must hand the folded query to the wrapper, not drop it.
+func TestDescribeAddonVersionsRoute_PassesTheQuery(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	got := captureDescribeAddonVersions(t, nc)
+
+	for _, r := range routes {
+		if r.Action != "DescribeAddonVersions" {
+			continue
+		}
+		_, err := r.Handler(context.Background(), nc, acct, "", nil, []byte(`{"addonName":["argocd"]}`))
+		require.NoError(t, err)
+		assert.Equal(t, "argocd", aws.StringValue(got().AddonName))
+		return
+	}
+	t.Fatal("no DescribeAddonVersions route")
+}

spinifex/domains/eks/awsapi/addons.go

@@ -16,8 +16,23 @@ func ListAddons(ctx context.Context, natsConn *nats.Conn, accountID, cluster str
 }
 
 // DescribeAddonVersions — GET /addons/supported-versions.
-func DescribeAddonVersions(ctx context.Context, natsConn *nats.Conn, accountID string) (*eks.DescribeAddonVersionsOutput, error) {
-	return ekscluster.NewNATSEKSService(natsConn).DescribeAddonVersions(ctx, &eks.DescribeAddonVersionsInput{}, accountID)
+func DescribeAddonVersions(ctx context.Context, natsConn *nats.Conn, accountID string, query []byte) (*eks.DescribeAddonVersionsOutput, error) {
+	q, err := parseListQuery(query)
+	if err != nil {
+		return nil, err
+	}
+	input := &eks.DescribeAddonVersionsInput{
+		AddonName:         q.str("addonName"),
+		KubernetesVersion: q.str("kubernetesVersion"),
+		NextToken:         q.str("nextToken"),
+		Types:             q.strs("types"),
+		Publishers:        q.strs("publishers"),
+		Owners:            q.strs("owners"),
+	}
+	if input.MaxResults, err = q.int64("maxResults"); err != nil {
+		return nil, err
+	}
+	return ekscluster.NewNATSEKSService(natsConn).DescribeAddonVersions(ctx, input, accountID)
 }
 
 // CreateAddon — POST /clusters/{name}/addons.

spinifex/domains/eks/awsapi/query.go

@@ -0,0 +1,61 @@
+package awsapi
+
+import (
+	"encoding/json"
+	"strconv"
+
+	"github.com/aws/aws-sdk-go/aws"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// listQuery holds the query-string inputs of a GET action, which the gateway
+// folds into the body as a JSON object of string lists.
+type listQuery map[string][]string
+
+// parseListQuery decodes the folded query of a GET action. An empty body means
+// the request carried no query string.
+func parseListQuery(body []byte) (listQuery, error) {
+	q := listQuery{}
+	if len(body) == 0 {
+		return q, nil
+	}
+	if err := json.Unmarshal(body, (*map[string][]string)(&q)); err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "malformed query string")
+	}
+	return q, nil
+}
+
+// str returns the first value of name, or nil when it is absent or empty.
+func (q listQuery) str(name string) *string {
+	if v := q[name]; len(v) > 0 && v[0] != "" {
+		return aws.String(v[0])
+	}
+	return nil
+}
+
+// strs returns every non-empty value of name, or nil when there are none.
+func (q listQuery) strs(name string) []*string {
+	var out []string
+	for _, v := range q[name] {
+		if v != "" {
+			out = append(out, v)
+		}
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return aws.StringSlice(out)
+}
+
+// int64 returns the first value of name as an integer, or nil when absent.
+func (q listQuery) int64(name string) (*int64, error) {
+	s := q.str(name)
+	if s == nil {
+		return nil, nil
+	}
+	n, err := strconv.ParseInt(*s, 10, 64)
+	if err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "%s must be an integer", name)
+	}
+	return aws.Int64(n), nil
+}

spinifex/domains/eks/awsapi/registration.go

@@ -82,17 +82,39 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	if err == nil {
 		return nil
 	}
-	if action, _, _, ok := router.Lookup(inv.Request.Method, inv.Request.URL.EscapedPath()); ok && IsInternalAction(action) {
+	action, _, _, routed := router.Lookup(inv.Request.Method, inv.Request.URL.EscapedPath())
+	if routed && IsInternalAction(action) {
 		return err
 	}
 	code, msg, ok := awserrors.ResolveErrorDetail(err)
 	if !ok || code != awserrors.ErrorInvalidParameterValue {
 		return err
 	}
-	if msg == "" {
-		return errors.New(awserrors.ErrorEKSInvalidParameter)
+	wire := awserrors.ErrorEKSInvalidParameter
+	if override, ok := parameterRefusalCodes[action]; ok {
+		wire = override
+		if msg == "" {
+			msg = "a parameter of the request is not valid"
+		}
 	}
-	return awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "%s", msg)
+	if msg == "" {
+		return errors.New(wire)
+	}
+	return awserrors.Errorf(wire, "%s", msg)
+}
+
+// parameterRefusalCodes holds the operations whose model does not declare
+// InvalidParameterException, with the declared code a refused parameter uses.
+var parameterRefusalCodes = map[string]string{
+	"DescribeCluster":              awserrors.ErrorEKSClient,
+	"DeleteCluster":                awserrors.ErrorEKSClient,
+	"DescribeAccessEntry":          awserrors.ErrorEKSInvalidRequest,
+	"DeleteAccessEntry":            awserrors.ErrorEKSInvalidRequest,
+	"DisassociateAccessPolicy":     awserrors.ErrorEKSInvalidRequest,
+	"ListAssociatedAccessPolicies": awserrors.ErrorEKSInvalidRequest,
+	"TagResource":                  awserrors.ErrorEKSBadRequest,
+	"UntagResource":                awserrors.ErrorEKSBadRequest,
+	"ListTagsForResource":          awserrors.ErrorEKSBadRequest,
 }
 
 // serve resolves the route, authorizes the request, then runs the action.

spinifex/domains/eks/awsapi/registration_error_codes_test.go

@@ -0,0 +1,98 @@
+//test:in-package — routes is unexported, and the table must cover exactly the
+// actions the registration serves.
+
+package awsapi
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/bluebottle/pkg/iampolicy"
+	"github.com/mulgadc/spinifex/internal/awsmodel"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// routePath fills a route pattern with placeholder segments. The tag routes
+// take an ARN, which has to be escaped to stay one path segment.
+func routePath(pattern string) string {
+	arn := url.PathEscape("arn:aws:eks:us-east-1:111122223333:cluster/c")
+	path := strings.ReplaceAll(pattern, "*", arn)
+	for {
+		start := strings.Index(path, "{")
+		if start < 0 {
+			return path
+		}
+		end := strings.Index(path[start:], "}")
+		path = path[:start] + "x" + path[start+end+1:]
+	}
+}
+
+// A parameter refusal must reach the wire under a code the operation declares
+// in the EKS model, or SDK clients cannot match it to a typed error.
+func TestRegistration_ParameterRefusalCodeIsDeclaredByOperation(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	sub, err := nc.Subscribe("eks.>", func(m *nats.Msg) {
+		if m.Reply != "" {
+			_ = m.Respond(awserrors.GenerateErrorPayload(awserrors.ErrorInvalidParameterValue))
+		}
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	model, err := awsmodel.Load(awsmodel.EKS)
+	require.NoError(t, err)
+	registration := NewRegistration(Deps{NATS: nc})
+
+	for _, route := range routes {
+		if IsInternalAction(route.Action) {
+			continue
+		}
+		// The handler takes no input and the model declares no client error.
+		if route.Action == "ListAccessPolicies" {
+			continue
+		}
+		t.Run(route.Action, func(t *testing.T) {
+			operation, ok := model.Operation(route.Action)
+			require.True(t, ok, "%s is not an operation in the EKS model", route.Action)
+			declared := make([]string, 0, len(operation.Errors))
+			for _, ref := range operation.Errors {
+				declared = append(declared, ref.Shape)
+			}
+
+			req := httptest.NewRequest(route.Method, routePath(route.Pattern), strings.NewReader(`{}`))
+			inv := dispatch.Invocation{
+				Request:   req,
+				AccountID: "111122223333",
+				Region:    "us-east-1",
+				Authorize: func(string, string, []string, iampolicy.ConditionKeys) error { return nil },
+				Actor: func() (dispatch.Actor, error) {
+					return dispatch.Actor{AccountID: "111122223333", PrincipalType: "user", UserName: "alice"}, nil
+				},
+			}
+
+			err := registration.Dispatch(httptest.NewRecorder(), inv)
+
+			require.Error(t, err)
+			code, ok := awserrors.ResolveErrorCode(err)
+			require.True(t, ok, "unresolvable error code: %v", err)
+			assert.Contains(t, declared, code,
+				"%s %s refuses a parameter with %s, which the operation does not declare", route.Method, route.Pattern, code)
+		})
+	}
+}
+
+// The access operations declare InvalidRequestException, not a parameter code.
+func TestEKSInvalidRequestCodeIsRegistered(t *testing.T) {
+	assert.Equal(t, "InvalidRequestException", awserrors.ErrorEKSInvalidRequest)
+	assert.Equal(t, http.StatusBadRequest, awserrors.HTTPStatusForError(awserrors.Errorf(awserrors.ErrorEKSInvalidRequest, "x")))
+	assert.Equal(t, "BadRequestException", awserrors.ErrorEKSBadRequest)
+	assert.Equal(t, http.StatusBadRequest, awserrors.HTTPStatusForError(awserrors.Errorf(awserrors.ErrorEKSBadRequest, "x")))
+}

spinifex/domains/eks/awsapi/routes.go

@@ -131,7 +131,7 @@ var routes = []route{
 	// Addons
 	{Method: "GET", Pattern: "/addons/supported-versions", Action: "DescribeAddonVersions",
 		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {
-			return DescribeAddonVersions(ctx, nc, acct)
+			return DescribeAddonVersions(ctx, nc, acct, b)
 		}},
 	{Method: "GET", Pattern: "/clusters/{clusterName}/addons", Action: "ListAddons",
 		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {

spinifex/domains/eks/awsapi/wrappers_test.go

@@ -176,7 +176,7 @@ func TestGatewayWrappers_Addons(t *testing.T) {
 	require.NoError(t, err)
 	assert.NotNil(t, out1)
 
-	out2, err := DescribeAddonVersions(context.Background(), nc, acct)
+	out2, err := DescribeAddonVersions(context.Background(), nc, acct, nil)
 	require.NoError(t, err)
 	assert.NotNil(t, out2)
 

spinifex/domains/eks/cluster/addon_describe_versions_test.go

@@ -0,0 +1,171 @@
+package cluster
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/domains/eks/addon"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func describeVersions(t *testing.T, svc *EKSServiceImpl, in *eks.DescribeAddonVersionsInput) *eks.DescribeAddonVersionsOutput {
+	t.Helper()
+	out, err := svc.DescribeAddonVersions(context.Background(), in, testAccountID)
+	require.NoError(t, err)
+	return out
+}
+
+// The terraform aws_eks_addon_version data source asks for one add-on at the
+// cluster's Kubernetes version, then takes the version whose compatibility for
+// that cluster version is the default (or, with most_recent, the first one).
+func TestDescribeAddonVersions_TerraformProviderCall(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	out := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{
+		AddonName:         aws.String(csiAddon),
+		KubernetesVersion: aws.String(defaultK8sVersion),
+	})
+	require.Len(t, out.Addons, 1)
+	info := out.Addons[0]
+	assert.Equal(t, csiAddon, aws.StringValue(info.AddonName))
+	assert.Nil(t, out.NextToken)
+
+	spec, ok := addon.Lookup(csiAddon)
+	require.True(t, ok)
+	require.Equal(t, spec.AWSVersions(), versionStrings(info), "newest first, AWS form")
+
+	defaults := 0
+	for _, v := range info.AddonVersions {
+		require.Len(t, v.Compatibilities, 1, "one entry, for the requested cluster version")
+		c := v.Compatibilities[0]
+		assert.Equal(t, defaultK8sVersion, aws.StringValue(c.ClusterVersion))
+		assert.Equal(t, []string{"*"}, aws.StringValueSlice(c.PlatformVersions))
+		if aws.BoolValue(c.DefaultVersion) {
+			defaults++
+			assert.Equal(t, spec.AWSVersion(spec.DefaultVersion), aws.StringValue(v.AddonVersion))
+		}
+	}
+	assert.Equal(t, 1, defaults, "exactly one defaultVersion true")
+}
+
+// AWS observed (EKS-01): compatibilities holds only the requested version. The
+// response for a version no add-on supports is not captured; this pins the
+// shape #61 chose, an empty list with no error.
+func TestDescribeAddonVersions_UnsupportedKubernetesVersionIsEmpty(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	out := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{
+		AddonName:         aws.String(csiAddon),
+		KubernetesVersion: aws.String("1.29"),
+	})
+	assert.Empty(t, out.Addons)
+	assert.Nil(t, out.NextToken)
+}
+
+func TestDescribeAddonVersions_FiltersByNameAndHidesFixturesUnlessNamed(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	for _, name := range []string{csiAddon, "argocd", albController} {
+		out := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{AddonName: aws.String(name)})
+		require.Len(t, out.Addons, 1, name)
+		assert.Equal(t, name, aws.StringValue(out.Addons[0].AddonName))
+	}
+	assert.Empty(t, describeVersions(t, svc, &eks.DescribeAddonVersionsInput{AddonName: aws.String("vpc-cni")}).Addons)
+}
+
+// maxResults counts add-ons, not versions; a page boundary never splits an
+// add-on's version list, and the last page carries no token.
+func TestDescribeAddonVersions_PagesByAddon(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	all := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{})
+	require.Greater(t, len(all.Addons), 1)
+	assert.Nil(t, all.NextToken)
+
+	var names []string
+	in := &eks.DescribeAddonVersionsInput{MaxResults: aws.Int64(1)}
+	for range len(all.Addons) {
+		out := describeVersions(t, svc, in)
+		require.Len(t, out.Addons, 1)
+		names = append(names, aws.StringValue(out.Addons[0].AddonName))
+		assert.Equal(t, versionStrings(all.Addons[len(names)-1]), versionStrings(out.Addons[0]))
+		in.NextToken = out.NextToken
+	}
+	assert.Nil(t, in.NextToken, "last page carries no token")
+	var want []string
+	for _, a := range all.Addons {
+		want = append(want, aws.StringValue(a.AddonName))
+	}
+	assert.Equal(t, want, names)
+}
+
+// Every add-on carries the descriptive fields the data source and console read.
+func TestDescribeAddonVersions_CarriesAddonAndVersionMetadata(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	for _, a := range describeVersions(t, svc, &eks.DescribeAddonVersionsInput{}).Addons {
+		name := aws.StringValue(a.AddonName)
+		assert.NotEmpty(t, aws.StringValue(a.Type), name)
+		assert.NotEmpty(t, aws.StringValue(a.Owner), name)
+		assert.NotEmpty(t, aws.StringValue(a.Publisher), name)
+		assert.Nil(t, a.MarketplaceInformation, name)
+		for _, v := range a.AddonVersions {
+			assert.NotEmpty(t, v.Architecture, name)
+			assert.NotNil(t, v.RequiresConfiguration, name)
+			assert.NotNil(t, v.RequiresIamPermissions, name)
+		}
+	}
+}
+
+// types, publishers and owners are model filters; each keeps add-ons whose
+// field matches one of the given values.
+func TestDescribeAddonVersions_FiltersByTypePublisherOwner(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	first := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{}).Addons[0]
+	cases := []struct {
+		name  string
+		match func(*eks.DescribeAddonVersionsInput, []*string)
+		value *string
+	}{
+		{"types", func(in *eks.DescribeAddonVersionsInput, v []*string) { in.Types = v }, first.Type},
+		{"publishers", func(in *eks.DescribeAddonVersionsInput, v []*string) { in.Publishers = v }, first.Publisher},
+		{"owners", func(in *eks.DescribeAddonVersionsInput, v []*string) { in.Owners = v }, first.Owner},
+	}
+	for _, c := range cases {
+		hit := &eks.DescribeAddonVersionsInput{}
+		c.match(hit, aws.StringSlice([]string{"nobody", aws.StringValue(c.value)}))
+		out := describeVersions(t, svc, hit)
+		require.NotEmpty(t, out.Addons, c.name)
+		assert.Contains(t, addonNames(out), aws.StringValue(first.AddonName), c.name)
+
+		miss := &eks.DescribeAddonVersionsInput{}
+		c.match(miss, aws.StringSlice([]string{"nobody"}))
+		assert.Empty(t, describeVersions(t, svc, miss).Addons, c.name)
+	}
+}
+
+func versionStrings(a *eks.AddonInfo) []string {
+	out := make([]string, 0, len(a.AddonVersions))
+	for _, v := range a.AddonVersions {
+		out = append(out, aws.StringValue(v.AddonVersion))
+	}
+	return out
+}
+
+func addonNames(o *eks.DescribeAddonVersionsOutput) []string {
+	out := make([]string, 0, len(o.Addons))
+	for _, a := range o.Addons {
+		out = append(out, aws.StringValue(a.AddonName))
+	}
+	return out
+}

spinifex/domains/eks/cluster/addon_version_format_test.go

@@ -0,0 +1,220 @@
+package cluster
+
+import (
+	"bytes"
+	"context"
+	"log/slog"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/domains/eks/addon"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	csiAddon       = "aws-ebs-csi-driver"
+	csiBare        = "1.40.1"
+	csiAWSVersion  = "v1.40.1-eksbuild.1"
+	csiWrongBuild  = "v1.40.1-eksbuild.2"
+	versionParam   = "addonVersion"
+	notBeforeAWSOK = "an AWS-format version must be accepted"
+)
+
+func TestDescribeAddonVersions_ReturnsAWSFormNewestFirst(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	out, err := svc.DescribeAddonVersions(context.Background(), &eks.DescribeAddonVersionsInput{}, testAccountID)
+	require.NoError(t, err)
+	require.NotEmpty(t, out.Addons)
+	for _, a := range out.Addons {
+		spec, ok := addon.Lookup(aws.StringValue(a.AddonName))
+		require.True(t, ok)
+		require.Len(t, a.AddonVersions, len(spec.Versions))
+		for i, v := range a.AddonVersions {
+			assert.Equal(t, "v"+spec.Versions[i]+"-eksbuild.1", aws.StringValue(v.AddonVersion),
+				"%s version %d", spec.Name, i)
+		}
+	}
+}
+
+func TestCreateAddon_AcceptsBothFormsAndStoresTheBareForm(t *testing.T) {
+	t.Parallel()
+	for name, input := range map[string]string{"aws form": csiAWSVersion, "bare form": csiBare} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			svc, fake := setupAddonService(t)
+
+			out, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+				AddonVersion: aws.String(input),
+			}, testAccountID)
+			require.NoError(t, err, notBeforeAWSOK)
+			assert.Equal(t, csiAWSVersion, aws.StringValue(out.Addon.AddonVersion), "response is in AWS form")
+
+			rec, err := addon.Get(t.Context(), acctKVForTest(t, svc), "c1", csiAddon)
+			require.NoError(t, err)
+			assert.Equal(t, csiBare, rec.AddonVersion, "the record keeps the upstream form")
+			require.Len(t, fake.installs, 1)
+			assert.Equal(t, csiBare, fake.installs[0].AddonVersion, "the bundle lookup sees the upstream form")
+		})
+	}
+}
+
+func TestCreateAddon_DefaultVersionResponseIsAWSForm(t *testing.T) {
+	t.Parallel()
+	svc, _ := setupAddonService(t)
+	out, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, csiAWSVersion, aws.StringValue(out.Addon.AddonVersion))
+}
+
+func TestUpdateAddon_AcceptsBothFormsAndStoresTheBareForm(t *testing.T) {
+	t.Parallel()
+	for name, input := range map[string]string{"aws form": csiAWSVersion, "bare form": csiBare} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			svc, fake := setupAddonService(t)
+			_, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+			}, testAccountID)
+			require.NoError(t, err)
+
+			_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+				AddonVersion: aws.String(input),
+			}, testAccountID)
+			require.NoError(t, err, notBeforeAWSOK)
+
+			rec, err := addon.Get(t.Context(), acctKVForTest(t, svc), "c1", csiAddon)
+			require.NoError(t, err)
+			assert.Equal(t, csiBare, rec.AddonVersion)
+			require.Len(t, fake.installs, 2)
+			assert.Equal(t, csiBare, fake.installs[1].AddonVersion)
+		})
+	}
+}
+
+func TestDescribeAddon_ReturnsAWSFormForAnyStoredForm(t *testing.T) {
+	t.Parallel()
+	for name, stored := range map[string]string{
+		"record stored bare before the change": csiBare,
+		"record already in AWS form":           csiAWSVersion,
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			svc, _ := setupAddonService(t)
+			_, err := svc.addons().Create(context.Background(), acctKVForTest(t, svc), testAccountID, addon.Desired{
+				Cluster: "c1", Name: csiAddon, Version: stored,
+			})
+			require.NoError(t, err)
+
+			desc, err := svc.DescribeAddon(context.Background(), &eks.DescribeAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+			}, testAccountID)
+			require.NoError(t, err)
+			assert.Equal(t, csiAWSVersion, aws.StringValue(desc.Addon.AddonVersion), "no double wrap")
+
+			list, err := svc.ListAddons(context.Background(), &eks.ListAddonsInput{ClusterName: aws.String("c1")}, testAccountID)
+			require.NoError(t, err)
+			assert.Equal(t, []string{csiAddon}, aws.StringValueSlice(list.Addons))
+		})
+	}
+}
+
+func TestUpdateAddon_ResponseAndRecordAfterBareRecord(t *testing.T) {
+	t.Parallel()
+	svc, _ := setupAddonService(t)
+	_, err := svc.addons().Create(context.Background(), acctKVForTest(t, svc), testAccountID, addon.Desired{
+		Cluster: "c1", Name: csiAddon, Version: csiBare,
+	})
+	require.NoError(t, err)
+
+	_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+		ConfigurationValues: aws.String(`{"a":1}`),
+	}, testAccountID)
+	require.NoError(t, err)
+	desc, err := svc.DescribeAddon(context.Background(), &eks.DescribeAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, csiAWSVersion, aws.StringValue(desc.Addon.AddonVersion))
+}
+
+func TestAddonVersionRefusal_NamesTheParameterAndListsSupportedValues(t *testing.T) {
+	t.Parallel()
+	svc, _ := setupAddonService(t)
+	_, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String("argocd"),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	refusals := map[string]func() error{
+		"create": func() error {
+			_, e := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+				AddonVersion: aws.String(csiWrongBuild),
+			}, testAccountID)
+			return e
+		},
+		"update": func() error {
+			_, e := svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String("argocd"),
+				AddonVersion: aws.String("v3.0.23-eksbuild.2"),
+			}, testAccountID)
+			return e
+		},
+	}
+	want := map[string]string{"create": csiAWSVersion, "update": "v3.0.23-eksbuild.1"}
+	for name, refuse := range refusals {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			err := refuse()
+			require.Error(t, err)
+			code, message, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok)
+			assert.Equal(t, awserrors.ErrorInvalidParameterValue, code, "this change leaves the error code alone")
+			assert.Contains(t, message, versionParam)
+			assert.Contains(t, message, want[name], "the supported AWS-form value is listed")
+		})
+	}
+}
+
+func TestAddonBareVersionInput_LogsItsUseAtInfo(t *testing.T) {
+	var buf bytes.Buffer
+	restore := slog.Default()
+	slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
+	t.Cleanup(func() { slog.SetDefault(restore) })
+
+	svc, _ := setupAddonService(t)
+	_, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	buf.Reset()
+	_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+		AddonVersion: aws.String(csiBare),
+	}, testAccountID)
+	require.NoError(t, err)
+	logged := buf.String()
+	assert.Contains(t, logged, "level=INFO")
+	assert.Contains(t, logged, "bare")
+	assert.Contains(t, logged, csiAddon)
+	assert.Contains(t, logged, "c1")
+
+	buf.Reset()
+	_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+		AddonVersion: aws.String(csiAWSVersion),
+	}, testAccountID)
+	require.NoError(t, err, notBeforeAWSOK)
+	assert.NotContains(t, buf.String(), "bare", "the AWS form is the intended form and logs nothing")
+}

spinifex/domains/eks/cluster/addons.go

@@ -2,8 +2,12 @@ package cluster
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
+	"log/slog"
+	"slices"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/eks"
@@ -35,15 +39,20 @@ func (s *EKSServiceImpl) ListAddons(ctx context.Context, input *eks.ListAddonsIn
 	return &eks.ListAddonsOutput{Addons: aws.StringSlice(names)}, nil
 }
 
-// DescribeAddonVersions returns the static add-on catalog, optionally filtered by name.
-func (s *EKSServiceImpl) DescribeAddonVersions(ctx context.Context, input *eks.DescribeAddonVersionsInput, _ string) (*eks.DescribeAddonVersionsOutput, error) {
-	filter := ""
-	if input != nil {
-		filter = aws.StringValue(input.AddonName)
+// DescribeAddonVersions returns the add-on catalogue filtered by name, owner,
+// publisher, type and the Kubernetes version its versions support, one page
+// of add-ons at a time.
+func (s *EKSServiceImpl) DescribeAddonVersions(_ context.Context, input *eks.DescribeAddonVersionsInput, _ string) (*eks.DescribeAddonVersionsOutput, error) {
+	if input == nil {
+		input = &eks.DescribeAddonVersionsInput{}
 	}
-	specs := addon.Specs()
-	out := make([]*eks.AddonInfo, 0, len(specs))
-	for _, spec := range specs {
+	// Every catalogue version supports only the K3s line the node image ships.
+	if v := aws.StringValue(input.KubernetesVersion); v != "" && v != defaultK8sVersion {
+		return &eks.DescribeAddonVersionsOutput{Addons: []*eks.AddonInfo{}}, nil
+	}
+	filter := aws.StringValue(input.AddonName)
+	var matched []addon.Spec
+	for _, spec := range addon.Specs() {
 		if filter != "" && spec.Name != filter {
 			continue
 		}
@@ -52,9 +61,51 @@ func (s *EKSServiceImpl) DescribeAddonVersions(ctx context.Context, input *eks.D
 		if spec.Hidden && filter == "" {
 			continue
 		}
+		if !matchesAny(input.Types, spec.Type) || !matchesAny(input.Owners, addonOwner) ||
+			!matchesAny(input.Publishers, addonPublisher) {
+			continue
+		}
+		matched = append(matched, spec)
+	}
+	page, next, err := pageAddons(matched, input.MaxResults, input.NextToken)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]*eks.AddonInfo, 0, len(page))
+	for _, spec := range page {
 		out = append(out, addonSpecToAWS(spec))
 	}
-	return &eks.DescribeAddonVersionsOutput{Addons: out}, nil
+	return &eks.DescribeAddonVersionsOutput{Addons: out, NextToken: next}, nil
+}
+
+// matchesAny reports whether the filter is empty or lists value.
+func matchesAny(filter []*string, value string) bool {
+	return len(filter) == 0 || slices.Contains(aws.StringValueSlice(filter), value)
+}
+
+// maxAddonPage bounds maxResults, the model's range being 1 to 100.
+const maxAddonPage = 100
+
+// pageAddons returns the add-ons after the token, at most maxResults of them.
+// The token is the base64 name of the last add-on returned, opaque to clients.
+func pageAddons(specs []addon.Spec, maxResults *int64, token *string) ([]addon.Spec, *string, error) {
+	if t := aws.StringValue(token); t != "" {
+		raw, err := base64.RawURLEncoding.DecodeString(t)
+		if err != nil {
+			return nil, nil, awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "nextToken is not valid")
+		}
+		after := string(raw)
+		specs = slices.DeleteFunc(slices.Clone(specs), func(sp addon.Spec) bool { return sp.Name <= after })
+	}
+	limit := len(specs)
+	if maxResults != nil {
+		limit = min(max(int(*maxResults), 1), maxAddonPage)
+	}
+	if limit >= len(specs) {
+		return specs, nil, nil
+	}
+	next := base64.RawURLEncoding.EncodeToString([]byte(specs[limit-1].Name))
+	return specs[:limit], &next, nil
 }
 
 // validateAddonServiceAccountRoleArn rejects a non-empty value that is not
@@ -89,11 +140,12 @@ func (s *EKSServiceImpl) CreateAddon(ctx context.Context, input *eks.CreateAddon
 	if !ok {
 		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 	}
-	version := aws.StringValue(input.AddonVersion)
-	if version == "" {
-		version = spec.DefaultVersion
-	} else if !spec.SupportsVersion(version) {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+	version := spec.DefaultVersion
+	if requested := aws.StringValue(input.AddonVersion); requested != "" {
+		var err error
+		if version, err = resolveAddonVersion(ctx, spec, requested, cluster); err != nil {
+			return nil, err
+		}
 	}
 	acctKV, err := s.acctKVForNewDependant(ctx, accountID, cluster)
 	if err != nil {
@@ -197,9 +249,13 @@ func (s *EKSServiceImpl) UpdateAddon(ctx context.Context, input *eks.UpdateAddon
 	version := aws.StringValue(input.AddonVersion)
 	if version != "" {
 		spec, ok := addon.Lookup(addonName)
-		if !ok || !spec.SupportsVersion(version) {
+		if !ok {
 			return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 		}
+		var err error
+		if version, err = resolveAddonVersion(ctx, spec, version, cluster); err != nil {
+			return nil, err
+		}
 	}
 	rec, err := s.addons().Update(ctx, acctKV, accountID, cluster, addonName, addon.Change{
 		Version:               version,
@@ -246,7 +302,7 @@ func addonRecordToAWS(cluster string, rec *addon.Record) *eks.Addon {
 	out := &eks.Addon{
 		AddonArn:     aws.String(rec.Arn),
 		AddonName:    aws.String(rec.AddonName),
-		AddonVersion: aws.String(rec.AddonVersion),
+		AddonVersion: aws.String(awsAddonVersion(rec)),
 		ClusterName:  aws.String(cluster),
 		Status:       aws.String(string(rec.Status)),
 		CreatedAt:    aws.Time(rec.CreatedAt),
@@ -269,17 +325,36 @@ func addonRecordToAWS(cluster string, rec *addon.Record) *eks.Addon {
 	return out
 }
 
-// addonSpecToAWS converts a catalog spec to the SDK AddonInfo shape.
+// The owner and publisher Spinifex reports for every catalogue add-on, as AWS
+// does for the add-ons it publishes.
+const (
+	addonOwner     = "aws"
+	addonPublisher = "eks"
+)
+
+// addonSpecToAWS converts a catalog spec to the SDK AddonInfo shape. Every
+// version supports the one cluster version the node image ships, and exactly
+// one of them is that cluster version's default.
 func addonSpecToAWS(spec addon.Spec) *eks.AddonInfo {
 	versions := make([]*eks.AddonVersionInfo, 0, len(spec.Versions))
-	for _, v := range spec.Versions {
+	for _, v := range spec.AWSVersions() {
 		versions = append(versions, &eks.AddonVersionInfo{
 			AddonVersion:           aws.String(v),
+			Architecture:           aws.StringSlice([]string{"amd64"}),
+			RequiresConfiguration:  aws.Bool(false),
 			RequiresIamPermissions: aws.Bool(spec.RequiresIRSA),
+			Compatibilities: []*eks.Compatibility{{
+				ClusterVersion:   aws.String(defaultK8sVersion),
+				PlatformVersions: aws.StringSlice([]string{"*"}),
+				DefaultVersion:   aws.Bool(v == spec.AWSVersion(spec.DefaultVersion)),
+			}},
 		})
 	}
 	return &eks.AddonInfo{
 		AddonName:     aws.String(spec.Name),
+		Type:          aws.String(spec.Type),
+		Owner:         aws.String(addonOwner),
+		Publisher:     aws.String(addonPublisher),
 		AddonVersions: versions,
 	}
 }
@@ -308,3 +383,30 @@ func (s *EKSServiceImpl) addonBucket(ctx context.Context, accountID string) (jet
 	}
 	return GetOrCreateAccountBucket(ctx, js, accountID)
 }
+
+// resolveAddonVersion maps a client version, in AWS form or bare upstream form,
+// to the upstream form the record and the bundle lookup use. The bare form is
+// more lenient than AWS, so each use is logged.
+func resolveAddonVersion(ctx context.Context, spec addon.Spec, requested, cluster string) (string, error) {
+	upstream, ok := spec.ParseVersion(requested)
+	if !ok {
+		return "", awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+			"addonVersion %q is not supported for add-on %s; supported versions: %s",
+			requested, spec.Name, strings.Join(spec.AWSVersions(), ", "))
+	}
+	if requested == upstream {
+		slog.InfoContext(ctx, "EKS add-on version given in bare upstream form",
+			"addon", spec.Name, "cluster", cluster, "version", requested)
+	}
+	return upstream, nil
+}
+
+// awsAddonVersion returns a stored record's version in AWS form; an add-on the
+// catalogue no longer lists keeps its stored string.
+func awsAddonVersion(rec *addon.Record) string {
+	spec, ok := addon.Lookup(rec.AddonName)
+	if !ok {
+		return rec.AddonVersion
+	}
+	return spec.AWSVersion(rec.AddonVersion)
+}

spinifex/domains/eks/cluster/addons_test.go

@@ -98,7 +98,9 @@ func TestCreateAddon_UnknownVersionRejected(t *testing.T) {
 		ClusterName: aws.String("c1"), AddonName: aws.String(albController),
 		AddonVersion: aws.String("9.9.9-nope"),
 	}, testAccountID)
-	require.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
 }
 
 func TestCreateAddon_CreatesStagingAndDefaultsVersion(t *testing.T) {
@@ -113,7 +115,7 @@ func TestCreateAddon_CreatesStagingAndDefaultsVersion(t *testing.T) {
 	require.NotNil(t, out.Addon)
 	// Version defaults to catalog default; status starts CREATING (honest until
 	// VM-side delivery confirms ACTIVE).
-	assert.Equal(t, spec.DefaultVersion, aws.StringValue(out.Addon.AddonVersion))
+	assert.Equal(t, spec.AWSVersion(spec.DefaultVersion), aws.StringValue(out.Addon.AddonVersion))
 	assert.Equal(t, string(addon.StatusCreating), aws.StringValue(out.Addon.Status))
 	assert.Contains(t, aws.StringValue(out.Addon.AddonArn), ":addon/c1/"+albController)
 
@@ -214,7 +216,7 @@ func TestUpdateAddon_ChangesVersionAndReinstalls(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 	assert.Equal(t, `{"replicaCount":2}`, aws.StringValue(desc.Addon.ConfigurationValues))
-	assert.Equal(t, spec.DefaultVersion, aws.StringValue(desc.Addon.AddonVersion))
+	assert.Equal(t, spec.AWSVersion(spec.DefaultVersion), aws.StringValue(desc.Addon.AddonVersion))
 	assert.Equal(t, string(addon.StatusUpdating), aws.StringValue(desc.Addon.Status))
 }
 
@@ -230,7 +232,9 @@ func TestUpdateAddon_UnknownVersionRejected(t *testing.T) {
 		ClusterName: aws.String("c1"), AddonName: aws.String("argocd"),
 		AddonVersion: aws.String("9.9.9-nope"),
 	}, testAccountID)
-	require.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
 }
 
 func TestUpdateAddon_MissingIsNotFound(t *testing.T) {

spinifex/domains/eks/cluster/cluster_security_group_test.go

@@ -0,0 +1,421 @@
+package cluster
+
+import (
+	"context"
+	"maps"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// sgCreatesIn returns the CreateSecurityGroup calls made against vpc whose name matches.
+func sgCreatesIn(f *fakeSGProvisioner, vpc string, match func(name string) bool) []*ec2.CreateSecurityGroupInput {
+	var out []*ec2.CreateSecurityGroupInput
+	for _, c := range f.createCalls {
+		if aws.StringValue(c.VpcId) == vpc && match(aws.StringValue(c.GroupName)) {
+			out = append(out, c)
+		}
+	}
+	return out
+}
+
+func sgCreateTags(c *ec2.CreateSecurityGroupInput) map[string]string {
+	m := map[string]string{}
+	for _, spec := range c.TagSpecifications {
+		maps.Copy(m, sgTagMap(spec.Tags))
+	}
+	return m
+}
+
+func authorizedOn(f *fakeSGProvisioner, sgID string) []*ec2.IpPermission {
+	var out []*ec2.IpPermission
+	for _, c := range f.authorizeCalls {
+		if aws.StringValue(c.GroupId) == sgID {
+			out = append(out, c.IpPermissions...)
+		}
+	}
+	return out
+}
+
+func deletedSGIDs(f *fakeSGProvisioner) []string {
+	var out []string
+	for _, c := range f.deleteCalls {
+		out = append(out, aws.StringValue(c.GroupId))
+	}
+	return out
+}
+
+// seedClusterSG registers an AWS-style cluster SG the way a created one looks.
+func seedClusterSG(f *fakeSGProvisioner, cluster, vpc, id string) {
+	f.existing[clusterSGNamePrefix+cluster+"-0123456789abcdef|"+vpc] = id
+	if f.tagsByID == nil {
+		f.tagsByID = map[string][]*ec2.Tag{}
+	}
+	f.tagsByID[id] = []*ec2.Tag{
+		{Key: aws.String("kubernetes.io/cluster/" + cluster), Value: aws.String("owned")},
+		{Key: aws.String("aws:eks:cluster-name"), Value: aws.String(cluster)},
+		{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(cluster)},
+		{Key: aws.String(clusterEKSRoleTagKey), Value: aws.String("cluster")},
+		{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)},
+	}
+}
+
+// seedCustomerSG registers a customer-made SG, optionally tagged to imitate ours.
+func seedCustomerSG(f *fakeSGProvisioner, name, vpc, id string, tg []*ec2.Tag) {
+	f.existing[name+"|"+vpc] = id
+	if f.tagsByID == nil {
+		f.tagsByID = map[string][]*ec2.Tag{}
+	}
+	f.tagsByID[id] = tg
+}
+
+// spoofTags are every tag a customer can set that the service also stamps.
+func spoofTags(cluster string) []*ec2.Tag {
+	return []*ec2.Tag{
+		{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(cluster)},
+		{Key: aws.String(clusterEKSRoleTagKey), Value: aws.String("cluster")},
+		{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)},
+		{Key: aws.String("kubernetes.io/cluster/" + cluster), Value: aws.String("owned")},
+	}
+}
+
+func setStoredClusterSG(t *testing.T, f *deleteClusterFixture, cluster, id string) {
+	t.Helper()
+	meta, err := GetClusterMeta(t.Context(), f.kv, cluster)
+	require.NoError(t, err)
+	meta.ResourcesVpcConfig.ClusterSecurityGroupId = id
+	require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta))
+}
+
+func createAlpha(t *testing.T, mutate func(*eks.CreateClusterInput)) (*eksServiceFixture, *ClusterMeta) {
+	t.Helper()
+	f := newEKSServiceFixture(t)
+	f.sg.uniqueIDs = true
+	in := createInput("alpha")
+	if mutate != nil {
+		mutate(in)
+	}
+	_, err := f.svc.CreateCluster(context.Background(), in, testAccountID, "")
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+	meta, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+	require.NoError(t, err)
+	require.NotNil(t, meta.ResourcesVpcConfig)
+	return f, meta
+}
+
+func withPrivateAccess(in *eks.CreateClusterInput) {
+	in.ResourcesVpcConfig.EndpointPrivateAccess = aws.Bool(true)
+}
+
+// CreateCluster makes the one AWS-style cluster SG in the caller's VPC and
+// reports it as clusterSecurityGroupId.
+func TestCreateCluster_ClusterSGHasAWSNameTagsAndRules(t *testing.T) {
+	t.Parallel()
+	f, meta := createAlpha(t, nil)
+
+	created := sgCreatesIn(f.sg, "vpc-aaa", func(n string) bool { return strings.HasPrefix(n, clusterSGNamePrefix+"alpha-") })
+	require.Len(t, created, 1, "exactly one eks-cluster-sg-<name>-<id> SG in the customer VPC")
+	name := aws.StringValue(created[0].GroupName)
+	assert.NotEmpty(t, strings.TrimPrefix(name, clusterSGNamePrefix+"alpha-"), "name carries a unique id suffix")
+
+	tg := sgCreateTags(created[0])
+	assert.Equal(t, "owned", tg["kubernetes.io/cluster/alpha"])
+	assert.Equal(t, "alpha", tg["aws:eks:cluster-name"])
+	assert.NotEmpty(t, tg["Name"])
+	assert.Equal(t, "alpha", tg[clusterEKSClusterTagKey])
+	assert.Equal(t, tags.ManagedByEKS, tg[tags.ManagedByKey])
+
+	id := f.sg.existing[name+"|vpc-aaa"]
+	require.NotEmpty(t, id)
+	assert.Equal(t, id, meta.ResourcesVpcConfig.ClusterSecurityGroupId)
+
+	perms := authorizedOn(f.sg, id)
+	require.Len(t, perms, 1, "inbound all from itself, nothing else")
+	assert.Equal(t, "-1", aws.StringValue(perms[0].IpProtocol))
+	require.Len(t, perms[0].UserIdGroupPairs, 1)
+	assert.Equal(t, id, aws.StringValue(perms[0].UserIdGroupPairs[0].GroupId))
+	for _, c := range f.sg.revokeEgressCall {
+		assert.NotEqual(t, id, aws.StringValue(c.GroupId), "egress stays allow-all")
+	}
+
+	out, err := f.svc.DescribeCluster(context.Background(), &eks.DescribeClusterInput{Name: aws.String("alpha")}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, id, aws.StringValue(out.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId))
+}
+
+// A new cluster has no customer-VPC control-plane or nodegroup SG, and exactly
+// one SG per cluster carries kubernetes.io/cluster/<name>.
+func TestCreateCluster_NoSeparateCustomerVPCControlPlaneOrNodegroupSG(t *testing.T) {
+	t.Parallel()
+	f, _ := createAlpha(t, withPrivateAccess)
+
+	retired := sgCreatesIn(f.sg, "vpc-aaa", func(n string) bool {
+		return n == ClusterControlPlaneSGName("alpha") || n == ClusterNodegroupSGName("alpha")
+	})
+	assert.Empty(t, retired, "customer VPC must not get control-plane-sg or nodegroup-sg")
+
+	tagged := 0
+	for _, c := range f.sg.createCalls {
+		if _, ok := sgCreateTags(c)["kubernetes.io/cluster/alpha"]; ok {
+			tagged++
+		}
+	}
+	assert.Equal(t, 1, tagged, "exactly one SG tagged kubernetes.io/cluster/alpha")
+}
+
+// The private-endpoint ENI carries its own SG and the cluster SG.
+func TestCreateCluster_PrivateEndpointENICarriesBothSGs(t *testing.T) {
+	t.Parallel()
+	f, meta := createAlpha(t, withPrivateAccess)
+
+	var eni *ec2.CreateNetworkInterfaceInput
+	for _, c := range f.vpc.createCalls {
+		if strings.Contains(aws.StringValue(c.Description), "private-endpoint") {
+			eni = c
+		}
+	}
+	require.NotNil(t, eni, "private-endpoint ENI created")
+	peSG := f.sg.existing[ClusterPrivateEndpointSGName("alpha")+"|vpc-aaa"]
+	require.NotEmpty(t, peSG)
+	assert.ElementsMatch(t,
+		[]string{peSG, meta.ResourcesVpcConfig.ClusterSecurityGroupId},
+		aws.StringValueSlice(eni.Groups))
+	assert.NotContains(t, sgTagMap(f.sg.tagsByID[peSG]), "kubernetes.io/cluster/alpha",
+		"only the cluster SG carries the cluster tag")
+}
+
+// The system control-plane VPC keeps its control-plane SG and rules.
+func TestCreateCluster_SystemCPVPCSecurityGroupUnchanged(t *testing.T) {
+	t.Parallel()
+	f, meta := createAlpha(t, nil)
+	require.NotNil(t, meta.ManagedCPVPC)
+	cpVPC := meta.ManagedCPVPC.VpcId
+
+	cp := sgCreatesIn(f.sg, cpVPC, func(n string) bool { return n == ClusterControlPlaneSGName("alpha") })
+	require.Len(t, cp, 1)
+	cpID := f.sg.existing[ClusterControlPlaneSGName("alpha")+"|"+cpVPC]
+	require.NotEmpty(t, cpID)
+
+	var ports []int64
+	selfRef := false
+	for _, p := range authorizedOn(f.sg, cpID) {
+		ports = append(ports, aws.Int64Value(p.FromPort))
+		for _, pair := range p.UserIdGroupPairs {
+			if aws.StringValue(pair.GroupId) == cpID {
+				selfRef = true
+			}
+		}
+	}
+	assert.Subset(t, ports, []int64{k3sAPIServerPort, konnectivityAgentPort, webhookReadyPort, 2379, 10250})
+	assert.True(t, selfRef, "HA self-referencing rules stay")
+	assert.Empty(t, sgCreatesIn(f.sg, cpVPC, func(n string) bool { return strings.HasPrefix(n, clusterSGNamePrefix) }),
+		"the cluster SG is a customer-VPC resource only")
+}
+
+// Workers launch with exactly the cluster SG and no extra SG or cross rules.
+func TestCreateNodegroup_WorkersGetExactlyTheClusterSG(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	f.sg.uniqueIDs = true
+	seedActiveClusterWithToken(t, f, "c1")
+	meta, err := GetClusterMeta(t.Context(), f.kv, "c1")
+	require.NoError(t, err)
+	meta.ResourcesVpcConfig.ClusterSecurityGroupId = "sg-cluster"
+	require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta))
+	seedClusterSG(f.sg, "c1", "vpc-aaa", "sg-cluster")
+	markWorkersReady(t, f, "c1", "ng1", 2)
+
+	_, err = f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 2), testAccountID)
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+
+	require.Len(t, f.worker.runCalls, 2)
+	for _, c := range f.worker.runCalls {
+		assert.Equal(t, []string{"sg-cluster"}, aws.StringValueSlice(c.SecurityGroupIds))
+	}
+	assert.Empty(t, f.sg.createCalls, "no nodegroup or control-plane SG is created for a worker")
+	assert.Empty(t, f.sg.authorizeCalls, "no CP<->nodegroup cross rules")
+}
+
+// A launch template that names security groups gets exactly those.
+func TestLaunchOneWorker_TemplateSecurityGroupsReplaceClusterSG(t *testing.T) {
+	t.Parallel()
+	reader := &fakeLaunchTemplates{out: &ec2.DescribeLaunchTemplateVersionsOutput{
+		LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{{
+			LaunchTemplateData: &ec2.ResponseLaunchTemplateData{
+				SecurityGroupIds: aws.StringSlice([]string{"sg-lt-1", "sg-lt-2"}),
+			},
+		}},
+	}}
+	f, err := launchWorkerWith(t, &NodegroupLaunchTemplate{ID: "lt-0123", Version: "1"}, reader)
+	require.NoError(t, err)
+
+	require.Len(t, f.worker.runCalls, 1)
+	assert.Equal(t, []string{"sg-lt-1", "sg-lt-2"}, aws.StringValueSlice(f.worker.runCalls[0].SecurityGroupIds))
+}
+
+// A template naming no security groups falls back to the cluster SG alone.
+func TestLaunchOneWorker_TemplateWithoutSecurityGroupsUsesClusterSG(t *testing.T) {
+	t.Parallel()
+	f, err := launchWorkerWith(t, &NodegroupLaunchTemplate{ID: "lt-0123", Version: "1"},
+		&fakeLaunchTemplates{out: templateVersionWith(nil)})
+	require.NoError(t, err)
+
+	require.Len(t, f.worker.runCalls, 1)
+	assert.Equal(t, []string{"sg-ng"}, aws.StringValueSlice(f.worker.runCalls[0].SecurityGroupIds))
+}
+
+// Delete removes the SG recorded on the cluster, whatever its name, and leaves
+// another cluster's SG in the same VPC alone.
+func TestDeleteCluster_RemovesClusterSGFoundByTag(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	seedClusterSG(f.sg, "alpha", "vpc-aaa", "sg-new")
+	seedClusterSG(f.sg, "beta", "vpc-aaa", "sg-beta")
+	setStoredClusterSG(t, f, "alpha", "sg-new")
+	f.sg.perms = map[string][]*ec2.IpPermission{"sg-new": {{
+		IpProtocol:       aws.String("-1"),
+		UserIdGroupPairs: []*ec2.UserIdGroupPair{{GroupId: aws.String("sg-new")}},
+	}}}
+	f.sg.enforceDepViolation = true
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	deleted := deletedSGIDs(f.sg)
+	assert.Contains(t, deleted, "sg-new")
+	assert.NotContains(t, deleted, "sg-beta")
+	_, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+	assert.Error(t, err, "teardown completed and removed the record")
+}
+
+// An old-layout cluster (control-plane-sg + nodegroup-sg, no cluster SG) still
+// has both SGs removed.
+func TestDeleteCluster_OldLayoutStillRemovesBothNamedSGs(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	f.sg.existing[ClusterControlPlaneSGName("alpha")+"|vpc-aaa"] = "sg-old-cp"
+	f.sg.existing[ClusterNodegroupSGName("alpha")+"|vpc-aaa"] = "sg-old-ng"
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	deleted := deletedSGIDs(f.sg)
+	assert.Contains(t, deleted, "sg-old-cp")
+	assert.Contains(t, deleted, "sg-old-ng")
+}
+
+// A cluster SG that cannot be deleted yet keeps the cluster DELETING; the
+// deleting reaper removes it once the dependency clears, not the request.
+func TestDeleteCluster_ClusterSGConvergesInTheReaper(t *testing.T) {
+	origBudget, origInterval := sgDeleteWaitBudget, sgDeleteWaitInterval
+	sgDeleteWaitBudget, sgDeleteWaitInterval = 10*time.Millisecond, time.Millisecond
+	t.Cleanup(func() { sgDeleteWaitBudget, sgDeleteWaitInterval = origBudget, origInterval })
+
+	f := newDeleteClusterFixture(t, "alpha")
+	seedClusterSG(f.sg, "alpha", "vpc-aaa", "sg-new")
+	setStoredClusterSG(t, f, "alpha", "sg-new")
+	f.sg.deleteFailsBefore = map[string]int{"sg-new": 1000}
+
+	deleteAndSettle(t, f.svc, "alpha")
+	meta := requireTeardownRetained(t, f.kv, "alpha")
+	assert.Contains(t, meta.DeleteFailedStep, stepSecurityGroups)
+
+	f.sg.mu.Lock()
+	f.sg.deleteFailsBefore["sg-new"] = 0
+	f.sg.mu.Unlock()
+	redrive(t, f.svc)
+
+	_, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+	assert.Error(t, err, "the reaper finished the teardown")
+}
+
+// A customer-made SG carrying the tags the service stamps is neither adopted by
+// a create retry nor deleted with the cluster.
+func TestEnsureClusterSG_DoesNotAdoptACustomerTaggedSG(t *testing.T) {
+	t.Parallel()
+	sg := newFakeSGProvisioner()
+	sg.uniqueIDs = true
+	seedCustomerSG(sg, "customer-sg", "vpc-aaa", "sg-spoof", spoofTags("c1"))
+
+	id, err := EnsureClusterSG(context.Background(), sg, testAccountID, "c1", "vpc-aaa")
+	require.NoError(t, err)
+
+	assert.NotEqual(t, "sg-spoof", id)
+	assert.Len(t, sg.createCalls, 1)
+	for _, p := range sg.authorizeCalls {
+		assert.NotEqual(t, "sg-spoof", aws.StringValue(p.GroupId))
+	}
+}
+
+func TestDeleteCluster_DoesNotDeleteACustomerTaggedSG(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	seedClusterSG(f.sg, "alpha", "vpc-aaa", "sg-new")
+	setStoredClusterSG(t, f, "alpha", "sg-new")
+	seedCustomerSG(f.sg, "customer-sg", "vpc-aaa", "sg-spoof", spoofTags("alpha"))
+	f.sg.perms = map[string][]*ec2.IpPermission{"sg-spoof": {{
+		IpProtocol: aws.String("tcp"), FromPort: aws.Int64(22), ToPort: aws.Int64(22),
+		IpRanges: []*ec2.IpRange{{CidrIp: aws.String("10.0.0.0/8")}},
+	}}}
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	assert.Contains(t, deletedSGIDs(f.sg), "sg-new")
+	assert.NotContains(t, deletedSGIDs(f.sg), "sg-spoof")
+	for _, r := range f.sg.revokeCalls {
+		assert.NotEqual(t, "sg-spoof", aws.StringValue(r.GroupId), "no rule is revoked on it")
+	}
+}
+
+// Deleting a cluster revokes, on an unrelated SG in the VPC, only the rules that
+// reference a cluster-owned SG.
+func TestDeleteClusterSGs_UnrelatedSGKeepsItsOtherRules(t *testing.T) {
+	t.Parallel()
+	sgp := newFakeSGProvisioner()
+	seedClusterSG(sgp, "alpha", "vpc-aaa", "sg-new")
+	seedCustomerSG(sgp, "customer-sg", "vpc-aaa", "sg-other", nil)
+	ssh := &ec2.IpPermission{
+		IpProtocol: aws.String("tcp"), FromPort: aws.Int64(22), ToPort: aws.Int64(22),
+		IpRanges: []*ec2.IpRange{{CidrIp: aws.String("10.0.0.0/8")}},
+	}
+	mixed := &ec2.IpPermission{
+		IpProtocol: aws.String("tcp"), FromPort: aws.Int64(443), ToPort: aws.Int64(443),
+		IpRanges: []*ec2.IpRange{{CidrIp: aws.String("192.168.0.0/16")}},
+		UserIdGroupPairs: []*ec2.UserIdGroupPair{
+			{GroupId: aws.String("sg-new")},
+			{GroupId: aws.String("sg-peer")},
+		},
+	}
+	sgp.perms = map[string][]*ec2.IpPermission{"sg-other": {ssh, mixed}}
+	sgp.permsEgress = map[string][]*ec2.IpPermission{"sg-other": {groupRefPerm("tcp", 6443, "sg-new"), groupRefPerm("tcp", 80, "sg-peer")}}
+
+	require.NoError(t, DeleteClusterSGs(context.Background(), sgp, testAccountID, "alpha", "vpc-aaa", "sg-new"))
+
+	var revoked []*ec2.IpPermission
+	for _, r := range sgp.revokeCalls {
+		if aws.StringValue(r.GroupId) == "sg-other" {
+			revoked = append(revoked, r.IpPermissions...)
+		}
+	}
+	require.Len(t, revoked, 1, "only the rule referencing the cluster SG goes")
+	require.Len(t, revoked[0].UserIdGroupPairs, 1)
+	assert.Equal(t, "sg-new", aws.StringValue(revoked[0].UserIdGroupPairs[0].GroupId))
+	assert.Empty(t, revoked[0].IpRanges, "its CIDR sources are not part of the revoke")
+
+	var revokedEgress []*ec2.IpPermission
+	for _, r := range sgp.revokeEgressCall {
+		if aws.StringValue(r.GroupId) == "sg-other" {
+			revokedEgress = append(revokedEgress, r.IpPermissions...)
+		}
+	}
+	require.Len(t, revokedEgress, 1)
+	assert.Equal(t, "sg-new", aws.StringValue(revokedEgress[0].UserIdGroupPairs[0].GroupId))
+	assert.NotContains(t, deletedSGIDs(sgp), "sg-other")
+}

spinifex/domains/eks/cluster/create_cluster_test.go

@@ -419,37 +419,6 @@ func TestCreateCluster_BuildsManagedCPVPCUnderSystemAccount(t *testing.T) {
 	assert.True(t, tagged, "NAT gateway EIP must carry the ManagedBy=eks tag")
 }
 
-// The primary cluster SG (clusterSecurityGroupId) must be created in the
-// caller's own VPC, not the system-managed CP VPC the control-plane VM
-// actually lives in — otherwise the tenant's own DescribeSecurityGroups can
-// never resolve it. fakeSubnetResolver resolves every subnet to "vpc-aaa".
-func TestCreateCluster_ClusterSecurityGroupCreatedInCallerVPC(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-
-	_, err := f.svc.CreateCluster(context.Background(), createInput("alpha"), testAccountID, "")
-	require.NoError(t, err)
-	f.svc.WaitLaunches()
-
-	meta, err := GetClusterMeta(t.Context(), f.kv, "alpha")
-	require.NoError(t, err)
-	require.NotNil(t, meta.ResourcesVpcConfig)
-	assert.NotEmpty(t, meta.ResourcesVpcConfig.ClusterSecurityGroupId)
-
-	found := false
-	for _, call := range f.sg.createCalls {
-		if aws.StringValue(call.GroupName) == ClusterControlPlaneSGName("alpha") && aws.StringValue(call.VpcId) == "vpc-aaa" {
-			found = true
-		}
-	}
-	assert.True(t, found, "control-plane SG must be created in the caller's VPC (vpc-aaa)")
-
-	out, err := f.svc.DescribeCluster(context.Background(), &eks.DescribeClusterInput{Name: aws.String("alpha")}, testAccountID)
-	require.NoError(t, err)
-	require.NotNil(t, out.Cluster.ResourcesVpcConfig)
-	assert.Equal(t, meta.ResourcesVpcConfig.ClusterSecurityGroupId, aws.StringValue(out.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId))
-}
-
 // Security groups the caller explicitly passes on CreateCluster must come back
 // unchanged from DescribeCluster — never substituted with platform-created ids.
 func TestCreateCluster_CallerSecurityGroupIdsEchoedNotOverwritten(t *testing.T) {

spinifex/domains/eks/cluster/nodegroup.go

@@ -472,13 +472,9 @@ type nodegroupLaunchCtx struct {
 func (s *EKSServiceImpl) launchNodegroupInfra(ctx context.Context, lc nodegroupLaunchCtx) {
 	acctKV, accountID, cluster, ng, meta, rec := lc.acctKV, lc.accountID, lc.cluster, lc.ng, lc.meta, lc.rec
 
-	cpSGID, ngSGID, err := EnsureClusterSGs(ctx, s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId)
+	ngSGID, err := EnsureWorkerSG(ctx, s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId, meta.ResourcesVpcConfig.ClusterSecurityGroupId)
 	if err != nil {
-		s.markNodegroupFailed(ctx, acctKV, cluster, ng, "ensure cluster SGs: "+err.Error())
-		return
-	}
-	if err := EnsureNodegroupSGRules(ctx, s.deps.VPCSG, accountID, cluster, cpSGID, ngSGID); err != nil {
-		s.markNodegroupFailed(ctx, acctKV, cluster, ng, "ensure nodegroup SG rules: "+err.Error())
+		s.markNodegroupFailed(ctx, acctKV, cluster, ng, "ensure cluster SG: "+err.Error())
 		return
 	}
 
@@ -707,11 +703,15 @@ func (s *EKSServiceImpl) launchOneWorker(ctx context.Context, rec *NodegroupReco
 		}}
 	}
 
-	metadataOptions, err := s.workerMetadataOptions(ctx, rec, accountID)
+	metadataOptions, templateSGs, err := s.workerTemplateSettings(ctx, rec, accountID)
 	if err != nil {
 		return "", err
 	}
 	runInput.MetadataOptions = metadataOptions
+	// As on AWS, security groups named by the launch template replace the cluster SG.
+	if len(templateSGs) > 0 {
+		runInput.SecurityGroupIds = aws.StringSlice(templateSGs)
+	}
 
 	// Attach the node role's instance profile so IMDS serves the role to the ECR
 	// credential provider; without it worker pulls from the internal ECR get 401.
@@ -743,21 +743,23 @@ func (s *EKSServiceImpl) launchOneWorker(ctx context.Context, rec *NodegroupReco
 	return "", errors.New("run worker returned no instance id")
 }
 
-// workerMetadataOptions chooses a worker's IMDS options: fields a launch template
-// version sets win as given, and every unset field gets the managed nodegroup
-// default (IMDSv2 required, hop limit 2) so pods behind the node can reach IMDS.
-func (s *EKSServiceImpl) workerMetadataOptions(ctx context.Context, rec *NodegroupRecord, accountID string) (*ec2.InstanceMetadataOptionsRequest, error) {
+// workerTemplateSettings chooses a worker's IMDS options and security groups.
+// Metadata fields a launch template version sets win as given, and every unset
+// field gets the managed nodegroup default (IMDSv2 required, hop limit 2) so pods
+// behind the node can reach IMDS. The template's security group IDs, if any, are
+// returned for the caller to use in place of the cluster SG.
+func (s *EKSServiceImpl) workerTemplateSettings(ctx context.Context, rec *NodegroupRecord, accountID string) (*ec2.InstanceMetadataOptionsRequest, []string, error) {
 	managedDefault := &ec2.InstanceMetadataOptionsRequest{
 		HttpTokens:              aws.String(ec2.HttpTokensStateRequired),
 		HttpPutResponseHopLimit: aws.Int64(workerMetadataHopLimit),
 	}
 	if rec.LaunchTemplate == nil {
-		return managedDefault, nil
+		return managedDefault, nil, nil
 	}
 	if s.deps.LaunchTemplates == nil {
 		slog.ErrorContext(ctx, "EKS nodegroup: launch template named but no launch template service wired",
 			"nodegroup", rec.Name, "launchTemplate", rec.LaunchTemplate.ID+rec.LaunchTemplate.Name)
-		return nil, errors.New(awserrors.ErrorServerInternal)
+		return nil, nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
 	version := rec.LaunchTemplate.Version
@@ -772,14 +774,16 @@ func (s *EKSServiceImpl) workerMetadataOptions(ctx context.Context, rec *Nodegro
 	}
 	out, err := s.deps.LaunchTemplates.DescribeLaunchTemplateVersions(ctx, in, accountID)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	if len(out.LaunchTemplateVersions) == 0 || out.LaunchTemplateVersions[0].LaunchTemplateData == nil {
-		return nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound)
+		return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound)
 	}
-	opts := out.LaunchTemplateVersions[0].LaunchTemplateData.MetadataOptions
+	data := out.LaunchTemplateVersions[0].LaunchTemplateData
+	sgIDs := aws.StringValueSlice(data.SecurityGroupIds)
+	opts := data.MetadataOptions
 	if opts == nil {
-		return managedDefault, nil
+		return managedDefault, sgIDs, nil
 	}
 	req := &ec2.InstanceMetadataOptionsRequest{
 		HttpTokens:              opts.HttpTokens,
@@ -796,7 +800,7 @@ func (s *EKSServiceImpl) workerMetadataOptions(ctx context.Context, rec *Nodegro
 	if req.HttpPutResponseHopLimit == nil {
 		req.HttpPutResponseHopLimit = managedDefault.HttpPutResponseHopLimit
 	}
-	return req, nil
+	return req, sgIDs, nil
 }
 
 // selectWorkerHost picks the host for the next worker: the eligible host (one
@@ -886,7 +890,7 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 		return s.attachRoleToProfile(accountID, profileName, roleName,
 			aws.StringValue(out.InstanceProfile.Arn), len(out.InstanceProfile.Roles) > 0)
 	}
-	if err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
 		return "", fmt.Errorf("get instance profile %q: %w", profileName, err)
 	}
 
@@ -895,7 +899,7 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 	})
 	if err != nil {
 		// A racing worker launch may have created it first; re-read and converge.
-		if err.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMEntityAlreadyExists {
 			return "", fmt.Errorf("create instance profile %q: %w", profileName, err)
 		}
 		got, gerr := s.iamEnsurer().GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{
@@ -922,7 +926,7 @@ func (s *EKSServiceImpl) attachRoleToProfile(accountID, profileName, roleName, p
 		InstanceProfileName: aws.String(profileName),
 		RoleName:            aws.String(roleName),
 	})
-	if err != nil && err.Error() != awserrors.ErrorIAMLimitExceeded {
+	if code, _ := awserrors.ResolveErrorCode(err); err != nil && code != awserrors.ErrorIAMLimitExceeded {
 		return "", fmt.Errorf("add role %q to instance profile %q: %w", roleName, profileName, err)
 	}
 	return profileARN, nil
@@ -1092,9 +1096,9 @@ func (s *EKSServiceImpl) launchWorkersCAS(ctx context.Context, acctKV jetstream.
 		}
 		return nil, fmt.Errorf("resolve eks-node AMI: %w", err)
 	}
-	_, ngSGID, err := EnsureClusterSGs(ctx, s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId)
+	ngSGID, err := EnsureWorkerSG(ctx, s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId, meta.ResourcesVpcConfig.ClusterSecurityGroupId)
 	if err != nil {
-		return nil, fmt.Errorf("ensure cluster SGs: %w", err)
+		return nil, fmt.Errorf("ensure cluster SG: %w", err)
 	}
 
 	// Each iteration records at most one worker, so bound by the gap plus CAS slack.

spinifex/domains/eks/cluster/nodegroup_test.go

@@ -338,6 +338,25 @@ func TestEnsureNodegroupSGRules_DuplicateRuleTolerated(t *testing.T) {
 		"duplicate-rule error must be treated as success")
 }
 
+// A cluster created before the cluster SG keeps launching workers into its
+// nodegroup SG with the control-plane cross rules.
+func TestCreateNodegroup_OldLayoutClusterKeepsNodegroupSG(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	f.sg.createIDs = []string{"sg-old-cp", "sg-old-ng"}
+	seedActiveClusterWithToken(t, f, "c1")
+	f.sg.existing[ClusterNodegroupSGName("c1")+"|vpc-aaa"] = "sg-old-ng"
+	markWorkersReady(t, f, "c1", "ng1", 1)
+
+	_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+
+	require.Len(t, f.worker.runCalls, 1)
+	assert.Equal(t, []string{"sg-old-ng"}, aws.StringValueSlice(f.worker.runCalls[0].SecurityGroupIds))
+	assert.Len(t, f.sg.authorizeCalls, 6, "the cp<->ng cross rules are still ensured")
+}
+
 func TestCreateNodegroup_HappyPath(t *testing.T) {
 	t.Parallel()
 	f := newEKSServiceFixture(t)
@@ -382,8 +401,14 @@ func TestCreateNodegroup_HappyPath(t *testing.T) {
 		}
 	}
 
-	// SG ingress rules were authorized.
+	// A cluster with no SG yet gets the cluster SG, created once with its self
+	// rule, and every worker carries it.
+	require.Len(t, f.sg.createCalls, 1)
+	clusterSG := f.sg.existing[aws.StringValue(f.sg.createCalls[0].GroupName)+"|vpc-aaa"]
 	assert.NotEmpty(t, f.sg.authorizeCalls)
+	for _, call := range f.worker.runCalls {
+		assert.Equal(t, []string{clusterSG}, aws.StringValueSlice(call.SecurityGroupIds))
+	}
 
 	// Duplicate create → ResourceInUseException.
 	_, err = f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 2), testAccountID)

spinifex/domains/eks/cluster/private_endpoint.go

@@ -30,7 +30,7 @@ func EnsurePrivateEndpointENI(ctx context.Context,
 	vpcSvc k3sVPCProvisioner,
 	sgp sgProvisioner,
 	resolver SubnetVPCResolver,
-	accountID, clusterName, subnetID, vpcID string,
+	accountID, clusterName, subnetID, vpcID, clusterSGID string,
 ) (*PrivateEndpointENI, error) {
 	if accountID == "" {
 		return nil, errors.New("eks: EnsurePrivateEndpointENI empty account id")
@@ -41,6 +41,9 @@ func EnsurePrivateEndpointENI(ctx context.Context,
 	if vpcID == "" {
 		return nil, errors.New("eks: EnsurePrivateEndpointENI empty vpc id")
 	}
+	if clusterSGID == "" {
+		return nil, errors.New("eks: EnsurePrivateEndpointENI empty cluster SG id")
+	}
 
 	vpcCIDR, err := resolver.GetVPCCIDR(ctx, accountID, vpcID)
 	if err != nil {
@@ -55,7 +58,7 @@ func EnsurePrivateEndpointENI(ctx context.Context,
 	eniOut, err := vpcSvc.CreateNetworkInterface(ctx, &ec2.CreateNetworkInterfaceInput{
 		SubnetId:    aws.String(subnetID),
 		Description: aws.String("EKS private-endpoint ENI for " + clusterName),
-		Groups:      aws.StringSlice([]string{sgID}),
+		Groups:      aws.StringSlice([]string{sgID, clusterSGID}),
 		TagSpecifications: []*ec2.TagSpecification{{
 			ResourceType: aws.String("network-interface"),
 			Tags: []*ec2.Tag{

spinifex/domains/eks/cluster/private_endpoint_test.go

@@ -132,30 +132,32 @@ func TestEnsurePrivateEndpointENI_HappyPath(t *testing.T) {
 	sgp := newFakeSGProvisioner()
 	sgp.createIDs = []string{"sg-pe-001"}
 
-	pe, err := EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "vpc-aaa")
+	pe, err := EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "vpc-aaa", "sg-cluster")
 	require.NoError(t, err)
 	assert.Equal(t, "eni-pe-001", pe.ENIID)
 	assert.Equal(t, "10.0.1.50", pe.ENIIP)
 	assert.Equal(t, "02:0a:01:23:45:67", pe.ENIMac)
 	assert.Equal(t, "sg-pe-001", pe.SGID)
 
-	// ENI must be created in the customer subnet, bound to the private-endpoint SG.
+	// ENI must be created in the customer subnet, bound to the private-endpoint SG
+	// and the cluster SG.
 	require.Len(t, vpcSvc.createCalls, 1)
 	createIn := vpcSvc.createCalls[0]
 	assert.Equal(t, "subnet-aaa", aws.StringValue(createIn.SubnetId))
-	require.Len(t, createIn.Groups, 1)
-	assert.Equal(t, "sg-pe-001", aws.StringValue(createIn.Groups[0]))
+	assert.Equal(t, []string{"sg-pe-001", "sg-cluster"}, aws.StringValueSlice(createIn.Groups))
 }
 
 func TestEnsurePrivateEndpointENI_EmptyInputsRejected(t *testing.T) {
 	t.Parallel()
 	vpcSvc := &fakeK3sVPC{}
 	sgp := newFakeSGProvisioner()
-	_, err := EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "", "alpha", "subnet-aaa", "vpc-aaa")
+	_, err := EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "", "alpha", "subnet-aaa", "vpc-aaa", "sg-cluster")
 	require.Error(t, err)
-	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "", "vpc-aaa")
+	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "", "vpc-aaa", "sg-cluster")
 	require.Error(t, err)
-	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "")
+	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "", "sg-cluster")
+	require.Error(t, err)
+	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "vpc-aaa", "")
 	require.Error(t, err)
 	assert.Empty(t, vpcSvc.createCalls)
 }

spinifex/domains/eks/cluster/security_groups.go

@@ -2,6 +2,8 @@ package cluster
 
 import (
 	"context"
+	"crypto/rand"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -40,6 +42,7 @@ const clusterEKSAccountTagKey = "spinifex:eks-cluster-account"
 const clusterEKSNodegroupTagKey = "spinifex:eks-nodegroup"
 
 const (
+	clusterEKSRoleCluster         = "cluster"
 	clusterEKSRoleControlPlane    = "control-plane"
 	clusterEKSRoleNodegroup       = "nodegroup"
 	clusterEKSRolePrivateEndpoint = "private-endpoint"
@@ -62,7 +65,101 @@ func ClusterNodegroupSGName(clusterName string) string {
 	return fmt.Sprintf("eks-cluster-%s-nodegroup-sg", clusterName)
 }
 
-// EnsureClusterSGs creates or reuses the control-plane and nodegroup SGs in the customer VPC.
+// clusterSGNamePrefix starts the name of the AWS-style cluster security group,
+// eks-cluster-sg-<cluster>-<id>.
+const clusterSGNamePrefix = "eks-cluster-sg-"
+
+// EnsureClusterSG creates or reuses the cluster security group in the customer
+// VPC: tagged as EKS tags its primary SG, inbound all from itself, outbound all
+// (the VPC service's default egress). It is found by tag, so the random name
+// suffix never has to be stored. Workers and the private-endpoint ENI carry it.
+func EnsureClusterSG(ctx context.Context, sgp sgProvisioner, accountID, clusterName, vpcID string) (string, error) {
+	if clusterName == "" {
+		return "", errors.New("eks: EnsureClusterSG empty cluster name")
+	}
+	if vpcID == "" {
+		return "", errors.New("eks: EnsureClusterSG empty vpc id")
+	}
+	id, err := lookupClusterSG(ctx, sgp, accountID, vpcID, clusterName)
+	if err != nil {
+		return "", err
+	}
+	if id == "" {
+		suffix := make([]byte, 8)
+		if _, err := rand.Read(suffix); err != nil {
+			return "", fmt.Errorf("generate cluster SG name: %w", err)
+		}
+		name := clusterSGNamePrefix + clusterName + "-" + hex.EncodeToString(suffix)
+		out, err := sgp.CreateSecurityGroup(ctx, &ec2.CreateSecurityGroupInput{
+			GroupName:   aws.String(name),
+			Description: aws.String(fmt.Sprintf("EKS created security group applied to ENI that is attached to EKS Control Plane master nodes, as well as any managed workloads (cluster %s)", clusterName)),
+			VpcId:       aws.String(vpcID),
+			TagSpecifications: []*ec2.TagSpecification{{
+				ResourceType: aws.String("security-group"),
+				Tags: []*ec2.Tag{
+					{Key: aws.String("Name"), Value: aws.String(name)},
+					{Key: aws.String("kubernetes.io/cluster/" + clusterName), Value: aws.String("owned")},
+					{Key: aws.String("aws:eks:cluster-name"), Value: aws.String(clusterName)},
+					{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)},
+					{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(clusterName)},
+					{Key: aws.String(clusterEKSRoleTagKey), Value: aws.String(clusterEKSRoleCluster)},
+				},
+			}},
+		}, accountID)
+		if err != nil {
+			return "", fmt.Errorf("create cluster SG %s: %w", name, err)
+		}
+		if out == nil || aws.StringValue(out.GroupId) == "" {
+			return "", fmt.Errorf("eks: CreateSecurityGroup returned empty GroupId for %s", name)
+		}
+		id = aws.StringValue(out.GroupId)
+	}
+	_, err = sgp.AuthorizeSecurityGroupIngress(ctx, &ec2.AuthorizeSecurityGroupIngressInput{
+		GroupId: aws.String(id),
+		IpPermissions: []*ec2.IpPermission{{
+			IpProtocol: aws.String("-1"),
+			UserIdGroupPairs: []*ec2.UserIdGroupPair{{
+				GroupId:     aws.String(id),
+				Description: aws.String("EKS cluster security group, all traffic from itself"),
+			}},
+		}},
+	}, accountID)
+	if err != nil && !awserrors.IsErrorCode(err, awserrors.ErrorInvalidPermissionDuplicate) {
+		return "", fmt.Errorf("authorize self ingress on cluster SG %s: %w", id, err)
+	}
+	return id, nil
+}
+
+// EnsureWorkerSG returns the SG nodegroup workers launch with. A cluster created
+// before the cluster SG existed keeps its nodegroup SG and control-plane rules
+// until it is recreated; every other cluster gets the SG recorded on it,
+// clusterSGID, or a freshly ensured one when none is recorded yet.
+func EnsureWorkerSG(ctx context.Context, sgp sgProvisioner, accountID, clusterName, vpcID, clusterSGID string) (string, error) {
+	if vpcID == "" {
+		return "", errors.New("eks: EnsureWorkerSG empty vpc id")
+	}
+	legacy, err := lookupSGByName(ctx, sgp, accountID, vpcID, ClusterNodegroupSGName(clusterName))
+	if err != nil {
+		return "", err
+	}
+	if legacy == "" {
+		if clusterSGID != "" {
+			return clusterSGID, nil
+		}
+		return EnsureClusterSG(ctx, sgp, accountID, clusterName, vpcID)
+	}
+	cpSGID, ngSGID, err := EnsureClusterSGs(ctx, sgp, accountID, clusterName, vpcID)
+	if err != nil {
+		return "", err
+	}
+	if err := EnsureNodegroupSGRules(ctx, sgp, accountID, clusterName, cpSGID, ngSGID); err != nil {
+		return "", err
+	}
+	return ngSGID, nil
+}
+
+// EnsureClusterSGs creates or reuses the control-plane and nodegroup SGs: the
+// control-plane SG in the system CP VPC, and both in a pre-cluster-SG customer VPC.
 // Idempotent; the two creates are independent (DeleteCluster handles partial teardown).
 func EnsureClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, clusterName, vpcID string) (controlPlaneSGID, nodegroupSGID string, err error) {
 	if clusterName == "" {
@@ -91,16 +188,16 @@ func EnsureClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, cluster
 	return controlPlaneSGID, nodegroupSGID, nil
 }
 
-// DeleteClusterSGs removes the cluster control-plane and nodegroup SGs. AWS
-// refuses to delete a security group still referenced by ANY other SG's rule —
-// not just the cp<->ng cross-reference (EnsureNodegroupSGRules) but the VPC's
-// LBC/ALB SG, the private-endpoint SG, and user/TF SGs, in either direction
-// (ingress or egress). To break every cycle order-independently, all ingress and
-// egress rules are revoked on every non-default SG in the VPC first, then the two
-// cluster-owned SGs are deleted. Referrer SGs are owned and deleted by their own
-// teardown — only the cluster SGs are deleted here. Missing SGs are no-ops; the
-// first error is returned after the full sweep.
-func DeleteClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, clusterName, vpcID string) error {
+// DeleteClusterSGs removes the cluster's SGs: the stored cluster SG, the
+// control-plane and nodegroup SGs of a cluster created before the cluster SG, and
+// the LBC-orphaned SGs. AWS refuses to delete an SG still referenced by another
+// SG's rule, in either direction, so before any delete every rule on a
+// cluster-owned SG is revoked and, on any other SG in the VPC, only the rules that
+// reference a cluster-owned SG. Unrelated customer SGs keep everything else.
+// clusterSGID is the id recorded on the cluster; ownership never comes from a
+// customer-settable tag. Missing SGs are no-ops; the first error is returned
+// after the full sweep.
+func DeleteClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, clusterName, vpcID, clusterSGID string) error {
 	if clusterName == "" {
 		return errors.New("eks: DeleteClusterSGs empty cluster name")
 	}
@@ -115,9 +212,6 @@ func DeleteClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, cluster
 		}
 	}
 
-	// Break every cross-reference in the VPC before deleting any SG. Revoking
-	// both directions on every non-default SG guarantees no rule still points at
-	// a cluster SG, so the deletes below succeed regardless of order.
 	out, err := sgp.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{
 		Filters: []*ec2.Filter{
 			{Name: aws.String("vpc-id"), Values: aws.StringSlice([]string{vpcID})},
@@ -127,12 +221,49 @@ func DeleteClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, cluster
 		slog.WarnContext(ctx, "DeleteClusterSGs: describe SGs failed", "vpc", vpcID, "err", err)
 		record(fmt.Errorf("describe SGs in vpc %s: %w", vpcID, err))
 	}
-	// A failed describe returns a nil out; fall back to an empty set so the two
-	// cross-reference sweeps below become no-ops while the targeted deletes still run.
+	// A failed describe returns a nil out; fall back to an empty set so the
+	// sweeps below become no-ops while the targeted deletes still run.
 	var sgs []*ec2.SecurityGroup
 	if out != nil {
 		sgs = out.SecurityGroups
 	}
+
+	// owned is every SG this cluster owns, in delete order: LBC-orphaned SGs
+	// first (they reference the cluster SG), then the stored cluster SG, then the
+	// pre-cluster-SG named ones.
+	var owned []string
+	isOwned := map[string]bool{}
+	own := func(id string) {
+		if id != "" && !isOwned[id] {
+			isOwned[id] = true
+			owned = append(owned, id)
+		}
+	}
+	for _, g := range sgs {
+		if g != nil && g.GroupId != nil && aws.StringValue(g.GroupName) != "default" && sgHasLBCClusterTag(g, clusterName) {
+			own(*g.GroupId)
+		}
+	}
+	own(clusterSGID)
+	// With no stored id (a create that died before recording it) the SG is still
+	// ours when it carries the tags only the service can set.
+	if clusterSGID == "" {
+		for _, g := range sgs {
+			if g != nil && g.GroupId != nil && sgIsClusterSG(g, clusterName) {
+				own(*g.GroupId)
+			}
+		}
+	}
+	for _, name := range []string{ClusterControlPlaneSGName(clusterName), ClusterNodegroupSGName(clusterName)} {
+		id, lookupErr := lookupSGByName(ctx, sgp, accountID, vpcID, name)
+		if lookupErr != nil {
+			slog.WarnContext(ctx, "DeleteClusterSGs: SG lookup failed", "sg", name, "err", lookupErr)
+			record(lookupErr)
+			continue
+		}
+		own(id)
+	}
+
 	for _, g := range sgs {
 		if g == nil || g.GroupId == nil {
 			continue
@@ -142,41 +273,19 @@ func DeleteClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, cluster
 		if aws.StringValue(g.GroupName) == "default" {
 			continue
 		}
-		if revokeErr := revokeAllSGRules(ctx, sgp, accountID, g); revokeErr != nil {
+		var revokeErr error
+		if isOwned[*g.GroupId] {
+			revokeErr = revokeAllSGRules(ctx, sgp, accountID, g)
+		} else {
+			revokeErr = revokeRulesReferencing(ctx, sgp, accountID, g, isOwned)
+		}
+		if revokeErr != nil {
 			slog.WarnContext(ctx, "DeleteClusterSGs: revoke rules failed", "sg", aws.StringValue(g.GroupId), "err", revokeErr)
 			record(revokeErr)
 		}
 	}
 
-	// Reap LBC-orphaned SGs (k8s-traffic-toc-*): the in-cluster AWS LoadBalancer
-	// Controller creates these in the customer VPC and spinifex never tracks them,
-	// so nothing else deletes them and they pin the VPC on DependencyViolation.
-	// Match by the LBC ownership tag; rules were already revoked above. Runs after
-	// the matching ALB reap, so the LB no longer references the SG.
-	for _, g := range sgs {
-		if g == nil || g.GroupId == nil || aws.StringValue(g.GroupName) == "default" {
-			continue
-		}
-		if !sgHasLBCClusterTag(g, clusterName) {
-			continue
-		}
-		if delErr := deleteSGAwaitingDetach(ctx, sgp, accountID, aws.StringValue(g.GroupId)); delErr != nil {
-			slog.WarnContext(ctx, "DeleteClusterSGs: LBC SG delete failed", "sg", aws.StringValue(g.GroupId), "err", delErr)
-			record(fmt.Errorf("delete LBC SG %s: %w", aws.StringValue(g.GroupId), delErr))
-		}
-	}
-
-	// Delete only the cluster-owned SGs.
-	for _, name := range []string{ClusterControlPlaneSGName(clusterName), ClusterNodegroupSGName(clusterName)} {
-		id, lookupErr := lookupSGByName(ctx, sgp, accountID, vpcID, name)
-		if lookupErr != nil {
-			slog.WarnContext(ctx, "DeleteClusterSGs: SG lookup failed", "sg", name, "err", lookupErr)
-			record(lookupErr)
-			continue
-		}
-		if id == "" {
-			continue
-		}
+	for _, id := range owned {
 		if delErr := deleteSGAwaitingDetach(ctx, sgp, accountID, id); delErr != nil {
 			slog.WarnContext(ctx, "DeleteClusterSGs: SG delete failed", "sg", id, "err", delErr)
 			record(fmt.Errorf("delete SG %s: %w", id, delErr))
@@ -248,6 +357,57 @@ func revokeAllSGRules(ctx context.Context, sgp sgProvisioner, accountID string,
 	return nil
 }
 
+// revokeRulesReferencing revokes, on a SG the cluster does not own, only the
+// rules that reference an owned SG, and only those references. Every other rule
+// on the SG is left in place.
+func revokeRulesReferencing(ctx context.Context, sgp sgProvisioner, accountID string, g *ec2.SecurityGroup, owned map[string]bool) error {
+	sgID := aws.StringValue(g.GroupId)
+	if perms := referencingPerms(g.IpPermissions, owned); len(perms) > 0 {
+		if _, err := sgp.RevokeSecurityGroupIngress(ctx, &ec2.RevokeSecurityGroupIngressInput{
+			GroupId:       g.GroupId,
+			IpPermissions: perms,
+		}, accountID); err != nil && !awserrors.IsNotFound(err) {
+			return fmt.Errorf("revoke ingress on %s: %w", sgID, err)
+		}
+	}
+	if perms := referencingPerms(g.IpPermissionsEgress, owned); len(perms) > 0 {
+		if _, err := sgp.RevokeSecurityGroupEgress(ctx, &ec2.RevokeSecurityGroupEgressInput{
+			GroupId:       g.GroupId,
+			IpPermissions: perms,
+		}, accountID); err != nil && !awserrors.IsNotFound(err) {
+			return fmt.Errorf("revoke egress on %s: %w", sgID, err)
+		}
+	}
+	return nil
+}
+
+// referencingPerms returns, for each permission that names an owned SG, a copy
+// carrying only those group references.
+func referencingPerms(perms []*ec2.IpPermission, owned map[string]bool) []*ec2.IpPermission {
+	var out []*ec2.IpPermission
+	for _, p := range perms {
+		if p == nil {
+			continue
+		}
+		var pairs []*ec2.UserIdGroupPair
+		for _, pair := range p.UserIdGroupPairs {
+			if pair != nil && owned[aws.StringValue(pair.GroupId)] {
+				pairs = append(pairs, pair)
+			}
+		}
+		if len(pairs) == 0 {
+			continue
+		}
+		out = append(out, &ec2.IpPermission{
+			IpProtocol:       p.IpProtocol,
+			FromPort:         p.FromPort,
+			ToPort:           p.ToPort,
+			UserIdGroupPairs: pairs,
+		})
+	}
+	return out
+}
+
 // EnsurePrivateEndpointSG creates (or reuses) the customer-VPC SG for the
 // private-endpoint ENI and admits the customer VPC CIDR on :443 (kubectl/SDK) and
 // :6443, so in-VPC workers + kubectl reach the apiserver via the Set A NIC. :6443
@@ -486,6 +646,42 @@ func sgHasLBCClusterTag(g *ec2.SecurityGroup, clusterName string) bool {
 	})
 }
 
+// sgIsClusterSG reports whether g carries every tag the service stamps on this
+// cluster's SG. aws:eks:cluster-name is reserved, so a customer cannot set it.
+func sgIsClusterSG(g *ec2.SecurityGroup, clusterName string) bool {
+	have := map[string]string{}
+	for _, tag := range g.Tags {
+		have[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value)
+	}
+	return have[tags.ManagedByKey] == tags.ManagedByEKS &&
+		have["aws:eks:cluster-name"] == clusterName &&
+		have[clusterEKSClusterTagKey] == clusterName &&
+		have[clusterEKSRoleTagKey] == clusterEKSRoleCluster
+}
+
+// lookupClusterSG finds a cluster SG that an earlier, interrupted create made
+// before its id was recorded. Only the tags the service alone can set count.
+func lookupClusterSG(ctx context.Context, sgp sgProvisioner, accountID, vpcID, clusterName string) (string, error) {
+	out, err := sgp.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{
+		Filters: []*ec2.Filter{
+			{Name: aws.String("vpc-id"), Values: aws.StringSlice([]string{vpcID})},
+			{Name: aws.String("tag:" + tags.ManagedByKey), Values: aws.StringSlice([]string{tags.ManagedByEKS})},
+			{Name: aws.String("tag:aws:eks:cluster-name"), Values: aws.StringSlice([]string{clusterName})},
+			{Name: aws.String("tag:" + clusterEKSClusterTagKey), Values: aws.StringSlice([]string{clusterName})},
+			{Name: aws.String("tag:" + clusterEKSRoleTagKey), Values: aws.StringSlice([]string{clusterEKSRoleCluster})},
+		},
+	}, accountID)
+	if err != nil {
+		return "", fmt.Errorf("describe cluster SG for %s: %w", clusterName, err)
+	}
+	for _, g := range out.SecurityGroups {
+		if g != nil && g.GroupId != nil && sgIsClusterSG(g, clusterName) {
+			return *g.GroupId, nil
+		}
+	}
+	return "", nil
+}
+
 func lookupSGByName(ctx context.Context, sgp sgProvisioner, accountID, vpcID, name string) (string, error) {
 	out, err := sgp.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{
 		Filters: []*ec2.Filter{

spinifex/domains/eks/cluster/security_groups_test.go

@@ -3,6 +3,7 @@ package cluster
 import (
 	"context"
 	"errors"
+	"strconv"
 	"strings"
 	"sync"
 	"testing"
@@ -52,6 +53,11 @@ type fakeSGProvisioner struct {
 	// nodegroup SG IDs.
 	createIDs []string
 
+	// uniqueIDs makes CreateSecurityGroup hand out sg-1, sg-2, ... once createIDs
+	// is exhausted, so a flow that creates several SGs can tell them apart.
+	uniqueIDs bool
+	nextSeq   int
+
 	createErr    error
 	describeErr  error
 	deleteErr    error
@@ -72,13 +78,23 @@ func (f *fakeSGProvisioner) CreateSecurityGroup(_ context.Context, input *ec2.Cr
 		return nil, f.createErr
 	}
 	id := "sg-default"
-	if len(f.createIDs) > 0 {
+	switch {
+	case len(f.createIDs) > 0:
 		id = f.createIDs[0]
 		f.createIDs = f.createIDs[1:]
+	case f.uniqueIDs:
+		f.nextSeq++
+		id = "sg-" + strconv.Itoa(f.nextSeq)
 	}
 	if input.GroupName != nil && input.VpcId != nil {
 		f.existing[*input.GroupName+"|"+*input.VpcId] = id
 	}
+	for _, spec := range input.TagSpecifications {
+		if f.tagsByID == nil {
+			f.tagsByID = map[string][]*ec2.Tag{}
+		}
+		f.tagsByID[id] = append(f.tagsByID[id], spec.Tags...)
+	}
 	return &ec2.CreateSecurityGroupOutput{GroupId: aws.String(id)}, nil
 }
 
@@ -104,15 +120,18 @@ func (f *fakeSGProvisioner) DescribeSecurityGroups(_ context.Context, input *ec2
 		return out, nil
 	}
 	var name, vpc string
+	tagFilters := map[string]string{}
 	for _, filt := range input.Filters {
 		if filt == nil || filt.Name == nil || len(filt.Values) == 0 || filt.Values[0] == nil {
 			continue
 		}
-		switch *filt.Name {
-		case "group-name":
+		switch {
+		case *filt.Name == "group-name":
 			name = *filt.Values[0]
-		case "vpc-id":
+		case *filt.Name == "vpc-id":
 			vpc = *filt.Values[0]
+		case strings.HasPrefix(*filt.Name, "tag:"):
+			tagFilters[strings.TrimPrefix(*filt.Name, "tag:")] = *filt.Values[0]
 		}
 	}
 	out := &ec2.DescribeSecurityGroupsOutput{}
@@ -123,6 +142,9 @@ func (f *fakeSGProvisioner) DescribeSecurityGroups(_ context.Context, input *ec2
 			if len(parts) != 2 || parts[1] != vpc {
 				continue
 			}
+			if !sgTagsMatch(f.tagsByID[id], tagFilters) {
+				continue
+			}
 			out.SecurityGroups = append(out.SecurityGroups, &ec2.SecurityGroup{
 				GroupId:             aws.String(id),
 				GroupName:           aws.String(parts[0]),
@@ -267,7 +289,7 @@ func TestDeleteClusterSGs_DeletesBothExisting(t *testing.T) {
 	sgp.existing["eks-cluster-alpha-control-plane-sg|vpc-aaa"] = "sg-existing-cp"
 	sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-existing-ng"
 
-	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 	require.NoError(t, err)
 
 	require.Len(t, sgp.deleteCalls, 2)
@@ -291,7 +313,7 @@ func TestDeleteClusterSGs_ReapsLBCTaggedSGs(t *testing.T) {
 		"sg-lbc-other": {{Key: aws.String(lbcClusterOwnershipTagKey), Value: aws.String("beta")}},
 	}
 
-	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 	require.NoError(t, err)
 
 	deleted := map[string]bool{}
@@ -309,7 +331,7 @@ func TestDeleteClusterSGs_MissingSGsNoOp(t *testing.T) {
 	t.Parallel()
 	sgp := newFakeSGProvisioner()
 
-	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 	require.NoError(t, err)
 	assert.Empty(t, sgp.deleteCalls, "no delete calls expected when SGs already absent")
 }
@@ -323,7 +345,7 @@ func TestDeleteClusterSGs_FirstErrorSurfacedSweepContinues(t *testing.T) {
 	// after a single attempt per SG (the retry path is covered separately).
 	sgp.deleteErr = errors.New("vpcd unavailable")
 
-	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "sg-existing-cp")
 	assert.Len(t, sgp.deleteCalls, 2, "both delete calls should be attempted despite first error")
@@ -348,7 +370,7 @@ func TestDeleteClusterSGs_RetriesDependencyViolation(t *testing.T) {
 		sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-ng"
 		sgp.deleteFailsBefore = map[string]int{"sg-ng": 2} // 2 DepViolations, then ok
 
-		err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+		err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 		require.NoError(t, err, "delete must succeed once the ENI detaches")
 		assert.GreaterOrEqual(t, len(sgp.deleteCalls), 3, "retried past the transient violations")
 	})
@@ -359,7 +381,7 @@ func TestDeleteClusterSGs_RetriesDependencyViolation(t *testing.T) {
 		sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-ng"
 		sgp.deleteErr = errors.New("DependencyViolation: still referenced")
 
-		err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+		err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 		require.Error(t, err, "a persistent DependencyViolation must surface, not leak the SG")
 		assert.Contains(t, err.Error(), "sg-ng")
 	})
@@ -382,7 +404,7 @@ func TestDeleteClusterSGs_RevokesCrossRefsBeforeDelete(t *testing.T) {
 	// AWS refuses to delete an SG still referenced by a sibling's rule.
 	sgp.enforceDepViolation = true
 
-	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 	require.NoError(t, err, "revoke-then-delete must clear the cp<->ng cycle so both SGs delete")
 
 	require.Len(t, sgp.revokeCalls, 2, "ingress on both SGs must be revoked before delete")
@@ -418,7 +440,7 @@ func TestDeleteClusterSGs_RevokesEgressAndNonClusterReferrers(t *testing.T) {
 	}
 	sgp.enforceDepViolation = true
 
-	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 	require.NoError(t, err, "revoking both directions on all non-default SGs must clear every cycle")
 
 	revokedEgress := map[string]bool{}
@@ -591,3 +613,64 @@ func assertSGCreateTagged(t *testing.T, in *ec2.CreateSecurityGroupInput, name,
 	assert.Equal(t, clusterName, got[clusterEKSClusterTagKey])
 	assert.Equal(t, role, got[clusterEKSRoleTagKey])
 }
+
+// sgTagsMatch reports whether tags carry every key=value in want.
+func sgTagsMatch(tags []*ec2.Tag, want map[string]string) bool {
+	have := sgTagMap(tags)
+	for k, v := range want {
+		if have[k] != v {
+			return false
+		}
+	}
+	return true
+}
+
+func sgTagMap(tags []*ec2.Tag) map[string]string {
+	m := map[string]string{}
+	for _, tg := range tags {
+		m[aws.StringValue(tg.Key)] = aws.StringValue(tg.Value)
+	}
+	return m
+}
+
+func TestEnsureClusterSG_EmptyInputsRejected(t *testing.T) {
+	t.Parallel()
+	sg := newFakeSGProvisioner()
+	_, err := EnsureClusterSG(context.Background(), sg, testAccountID, "", "vpc-aaa")
+	require.Error(t, err)
+	_, err = EnsureClusterSG(context.Background(), sg, testAccountID, "c1", "")
+	require.Error(t, err)
+	assert.Empty(t, sg.createCalls)
+}
+
+func TestEnsureClusterSG_SecondCallReusesTheTaggedSG(t *testing.T) {
+	t.Parallel()
+	sg := newFakeSGProvisioner()
+	sg.uniqueIDs = true
+
+	first, err := EnsureClusterSG(context.Background(), sg, testAccountID, "c1", "vpc-aaa")
+	require.NoError(t, err)
+	second, err := EnsureClusterSG(context.Background(), sg, testAccountID, "c1", "vpc-aaa")
+	require.NoError(t, err)
+
+	assert.Equal(t, first, second)
+	assert.Len(t, sg.createCalls, 1)
+}
+
+func TestEnsureClusterSG_DuplicateSelfRuleTolerated(t *testing.T) {
+	t.Parallel()
+	sg := newFakeSGProvisioner()
+	sg.authorizeErr = errors.New(awserrors.ErrorInvalidPermissionDuplicate)
+
+	_, err := EnsureClusterSG(context.Background(), sg, testAccountID, "c1", "vpc-aaa")
+	require.NoError(t, err)
+}
+
+func TestEnsureClusterSG_AuthorizeErrorSurfaced(t *testing.T) {
+	t.Parallel()
+	sg := newFakeSGProvisioner()
+	sg.authorizeErr = errors.New("boom")
+
+	_, err := EnsureClusterSG(context.Background(), sg, testAccountID, "c1", "vpc-aaa")
+	require.Error(t, err)
+}

spinifex/domains/eks/cluster/service_impl.go

@@ -977,13 +977,13 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc
 		return
 	}
 
-	// The customer-facing "cluster security group" AWS auto-creates lives in the
-	// caller's own VPC, not the system-managed CP VPC above — otherwise the
-	// tenant's own DescribeSecurityGroups can never resolve it. Idempotent with
-	// launchNodegroupInfra's own EnsureClusterSGs call, which reuses this same id.
-	custClusterSG, _, err := EnsureClusterSGs(ctx, s.deps.VPCSG, accountID, name, lc.vpcID)
+	// The customer-facing cluster security group lives in the caller's own VPC,
+	// not the system-managed CP VPC above, so the tenant's own
+	// DescribeSecurityGroups resolves it. Workers and the private-endpoint ENI
+	// carry it. Idempotent with the nodegroup launch, which finds it by tag.
+	custClusterSG, err := EnsureClusterSG(ctx, s.deps.VPCSG, accountID, name, lc.vpcID)
 	if err != nil {
-		s.failClusterLaunch(ctx, acctKV, name, accountID, meta, "ensure customer-vpc cluster SGs", err)
+		s.failClusterLaunch(ctx, acctKV, name, accountID, meta, "ensure customer-vpc cluster SG", err)
 		return
 	}
 	meta.ResourcesVpcConfig.ClusterSecurityGroupId = custClusterSG
@@ -1002,7 +1002,7 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc
 	// NIC and proxies to the CP target group with no data-plane change.
 	var crossAccountENIs []systeminstance.ExtraENIInput
 	if privateAccess {
-		pe, perr := EnsurePrivateEndpointENI(ctx, s.deps.VPCK3s, s.deps.VPCSG, s.deps.VPCSubnet, accountID, name, lc.subnetIDs[0], lc.vpcID)
+		pe, perr := EnsurePrivateEndpointENI(ctx, s.deps.VPCK3s, s.deps.VPCSG, s.deps.VPCSubnet, accountID, name, lc.subnetIDs[0], lc.vpcID, custClusterSG)
 		if perr != nil {
 			s.failClusterLaunch(ctx, acctKV, name, accountID, meta, "ensure private endpoint ENI", perr)
 			return
@@ -1621,7 +1621,7 @@ func (s *EKSServiceImpl) teardownClusterInfra(ctx context.Context, accountID, na
 		// New topology: SGs + IGW + subnets + route tables + NAT GW + VPC all live
 		// in the managed CP VPC under the system account. DeleteClusterCPVPC removes
 		// the IGW + VPC after the SGs, so SG cleanup must precede it.
-		cpSGErr := DeleteClusterSGs(ctx, s.deps.VPCSG, infraAcct, name, meta.ManagedCPVPC.VpcId)
+		cpSGErr := DeleteClusterSGs(ctx, s.deps.VPCSG, infraAcct, name, meta.ManagedCPVPC.VpcId, "")
 		if cpSGErr != nil {
 			fail(stepSecurityGroups, fmt.Errorf("delete cluster SGs: %w", cpSGErr))
 		}
@@ -1629,7 +1629,7 @@ func (s *EKSServiceImpl) teardownClusterInfra(ctx context.Context, accountID, na
 		// worker<->control-plane networking; reclaim them here too, or they orphan
 		// (cross-referencing each other) and pin the customer VPC on destroy.
 		if meta.ResourcesVpcConfig != nil && meta.ResourcesVpcConfig.VpcId != "" {
-			if err := DeleteClusterSGs(ctx, s.deps.VPCSG, accountID, name, meta.ResourcesVpcConfig.VpcId); err != nil {
+			if err := DeleteClusterSGs(ctx, s.deps.VPCSG, accountID, name, meta.ResourcesVpcConfig.VpcId, meta.ResourcesVpcConfig.ClusterSecurityGroupId); err != nil {
 				fail(stepSecurityGroups, fmt.Errorf("delete customer-VPC cluster SGs: %w", err))
 			}
 		}
@@ -1653,7 +1653,7 @@ func (s *EKSServiceImpl) teardownClusterInfra(ctx context.Context, accountID, na
 		// Legacy topology: CP lived in the customer VPC; reclaim its SGs + the
 		// cluster-owned IGW (ownership-scoped — a reused customer IGW is left
 		// intact). IGW delete must precede any VPC delete to avoid DependencyViolation.
-		if err := DeleteClusterSGs(ctx, s.deps.VPCSG, accountID, name, meta.ResourcesVpcConfig.VpcId); err != nil {
+		if err := DeleteClusterSGs(ctx, s.deps.VPCSG, accountID, name, meta.ResourcesVpcConfig.VpcId, meta.ResourcesVpcConfig.ClusterSecurityGroupId); err != nil {
 			fail(stepSecurityGroups, fmt.Errorf("delete cluster SGs: %w", err))
 		}
 		if s.deps.IGW != nil {

spinifex/domains/elbv2/haproxy.go

@@ -46,7 +46,7 @@ backend {{.Name}}
 {{- if .Redirect}}
     http-request redirect {{if .Redirect.Scheme}}scheme {{.Redirect.Scheme}}{{else}}location {{.Redirect.Location}}{{end}} code {{.Redirect.Code}}
 {{- else if .FixedResponse}}
-    http-request return status {{.FixedResponse.StatusCode}}{{if .FixedResponse.ContentType}} content-type "{{.FixedResponse.ContentType}}"{{end}}{{if .FixedResponse.Body}} string "{{.FixedResponse.Body}}"{{end}}
+    http-request return status {{.FixedResponse.StatusCode}}{{if .FixedResponse.ContentType}} content-type {{squote .FixedResponse.ContentType}}{{end}}{{if .FixedResponse.Body}} string {{squote .FixedResponse.Body}}{{end}}
 {{- else}}
     balance roundrobin{{if .HealthCheckPath}}
     option httpchk
@@ -65,7 +65,14 @@ backend {{.Name}}
 {{end}}
 `
 
-var haproxyHTTPTmpl = template.Must(template.New("haproxy").Parse(haproxyConfigTemplate))
+var haproxyHTTPTmpl = template.Must(template.New("haproxy").Funcs(template.FuncMap{"squote": haproxySingleQuote}).Parse(haproxyConfigTemplate))
+
+// haproxySingleQuote renders s as one literal HAProxy word. HAProxy expands
+// environment variables inside double quotes but not single quotes, so tenant
+// text is single-quoted with each embedded quote closed, escaped and reopened.
+func haproxySingleQuote(s string) string {
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
 
 // HAProxyConfig holds the full configuration for rendering the HAProxy template.
 type HAProxyConfig struct {
@@ -566,17 +573,10 @@ func validHTTPStatusCode(s string) bool {
 	return true
 }
 
-// validFixedResponseBody rejects bytes that would terminate or inject into the HAProxy directive.
+// validFixedResponseBody rejects bytes that would end the single-line HAProxy
+// directive the single-quoted body is rendered into.
 func validFixedResponseBody(s string) bool {
-	if s == "" {
-		return false
-	}
-	for _, r := range s {
-		if r == '\n' || r == '\r' || r == '"' || r == '\\' || r == '#' {
-			return false
-		}
-	}
-	return true
+	return s != "" && !strings.ContainsAny(s, "\n\r\x00")
 }
 
 // buildRedirect translates a RedirectAction into an HAProxy redirect directive.

spinifex/domains/elbv2/haproxy_test.go

@@ -279,7 +279,7 @@ func TestGenerateHAProxyConfig_FixedResponseDefault(t *testing.T) {
 	// Synthetic default backend named off the listener ARN, returning the fixed response.
 	assert.Contains(t, config, "default_backend bkdefault_lst-616760baf6a3031c3")
 	assert.Contains(t, config, "backend bkdefault_lst-616760baf6a3031c3")
-	assert.Contains(t, config, `http-request return status 404 content-type "text/plain" string "no route"`)
+	assert.Contains(t, config, `http-request return status 404 content-type 'text/plain' string 'no route'`)
 	// Host-header rule still routes to the app backend.
 	assert.Contains(t, config, "use_backend bk_tg-app1 if")
 	assert.Contains(t, config, "10.42.0.9:8080")
@@ -309,7 +309,59 @@ func TestGenerateHAProxyConfig_FixedResponseRejectsUnsafeBody(t *testing.T) {
 
 	assert.Contains(t, config, "http-request return status 200")
 	assert.NotContains(t, config, "always_true")
-	assert.NotContains(t, config, `string "evil`)
+	assert.NotContains(t, config, `string 'evil`)
+}
+
+// TestGenerateHAProxyConfig_FixedResponseIsLiteral renders the content type and
+// body in single quotes, where HAProxy performs no environment expansion, so a
+// body referencing the lb-agent's environment is served verbatim.
+func TestGenerateHAProxyConfig_FixedResponseIsLiteral(t *testing.T) {
+	t.Parallel()
+	lb := &LoadBalancerRecord{LoadBalancerID: "lb-env"}
+	listeners := []*ListenerRecord{
+		{
+			ListenerArn: "arn:lst-env",
+			Port:        80,
+			DefaultActions: []ListenerAction{
+				{Type: ActionTypeFixedResponse, FixedResponse: &FixedResponseAction{
+					StatusCode:  "200",
+					ContentType: "text/plain",
+					MessageBody: `sk=${LB_SECRET_KEY} ak=$LB_ACCESS_KEY it's "q" \ #x`,
+				}},
+			},
+		},
+	}
+
+	config, _, err := GenerateHAProxyConfigWithCerts(lb, listeners, nil, nil, "0.0.0.0", nil)
+	require.NoError(t, err)
+
+	assert.Contains(t, config, `http-request return status 200 content-type 'text/plain' string 'sk=${LB_SECRET_KEY} ak=$LB_ACCESS_KEY it'\''s "q" \ #x'`+"\n")
+	assert.NotContains(t, config, `"${LB_SECRET_KEY}`)
+}
+
+// TestGenerateHAProxyConfig_FixedResponseKeepsStoredTab keeps a stored body with
+// a tab, which is literal inside single quotes and rendered before API checks existed.
+func TestGenerateHAProxyConfig_FixedResponseKeepsStoredTab(t *testing.T) {
+	t.Parallel()
+	lb := &LoadBalancerRecord{LoadBalancerID: "lb-tab"}
+	listeners := []*ListenerRecord{
+		{
+			ListenerArn: "arn:lst-tab",
+			Port:        80,
+			DefaultActions: []ListenerAction{
+				{Type: ActionTypeFixedResponse, FixedResponse: &FixedResponseAction{
+					StatusCode:  "503",
+					ContentType: "text/html",
+					MessageBody: "<pre>\tdown</pre>",
+				}},
+			},
+		},
+	}
+
+	config, _, err := GenerateHAProxyConfigWithCerts(lb, listeners, nil, nil, "0.0.0.0", nil)
+	require.NoError(t, err)
+
+	assert.Contains(t, config, "http-request return status 503 content-type 'text/html' string '<pre>\tdown</pre>'\n")
 }
 
 // TestGenerateHAProxyConfig_RedirectSchemeDefault renders the canonical

spinifex/domains/elbv2/service_impl.go

@@ -17,6 +17,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"unicode"
+	"unicode/utf8"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -2701,8 +2703,9 @@ func listenerActionFromSDK(a *elbv2.Action) ListenerAction {
 	return action
 }
 
-// validateListenerAction enforces the per-type action contract. Redirect fields
-// are validated here so invalid input fails at the API, not at render time.
+// validateListenerAction enforces the per-type action contract. Redirect and
+// fixed-response fields are validated here so invalid input fails at the API,
+// not at render time.
 func validateListenerAction(a ListenerAction) error {
 	if a.Type == ActionTypeRedirect {
 		if a.Redirect == nil {
@@ -2710,6 +2713,39 @@ func validateListenerAction(a ListenerAction) error {
 		}
 		return validateRedirectAction(a.Redirect)
 	}
+	if a.Type == ActionTypeFixedResponse {
+		return validateFixedResponseAction(a.FixedResponse)
+	}
+	return nil
+}
+
+// fixedResponseContentTypes are the only ContentType values ELBv2 accepts.
+var fixedResponseContentTypes = map[string]bool{
+	"text/plain":             true,
+	"text/css":               true,
+	"text/html":              true,
+	"application/javascript": true,
+	"application/json":       true,
+}
+
+// maxFixedResponseBodyLen is ELBv2's MessageBody length limit in characters.
+const maxFixedResponseBodyLen = 1024
+
+// validateFixedResponseAction rejects a ContentType ELBv2 does not allow, or a
+// MessageBody that is too long or holds control characters.
+func validateFixedResponseAction(fr *FixedResponseAction) error {
+	if fr == nil {
+		return errors.New(awserrors.ErrorMissingParameter)
+	}
+	if fr.ContentType != "" && !fixedResponseContentTypes[fr.ContentType] {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	if utf8.RuneCountInString(fr.MessageBody) > maxFixedResponseBodyLen {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	if strings.ContainsFunc(fr.MessageBody, unicode.IsControl) {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
 	return nil
 }
 
@@ -3503,14 +3539,18 @@ func (s *ELBv2ServiceImpl) tgRecordToSDK(r *TargetGroupRecord, lbArns []string)
 		HealthCheckEnabled:         aws.Bool(r.HealthCheck.Enabled),
 		HealthCheckProtocol:        aws.String(r.HealthCheck.Protocol),
 		HealthCheckPort:            aws.String(r.HealthCheck.Port),
-		HealthCheckPath:            aws.String(r.HealthCheck.Path),
 		HealthCheckIntervalSeconds: aws.Int64(r.HealthCheck.IntervalSeconds),
 		HealthCheckTimeoutSeconds:  aws.Int64(r.HealthCheck.TimeoutSeconds),
 		HealthyThresholdCount:      aws.Int64(r.HealthCheck.HealthyThreshold),
 		UnhealthyThresholdCount:    aws.Int64(r.HealthCheck.UnhealthyThreshold),
-		Matcher: &elbv2.Matcher{
-			HttpCode: aws.String(r.HealthCheck.Matcher),
-		},
+	}
+
+	// AWS omits both keys on TCP target groups, which have no path or matcher.
+	if r.HealthCheck.Path != "" {
+		tg.HealthCheckPath = aws.String(r.HealthCheck.Path)
+	}
+	if r.HealthCheck.Matcher != "" {
+		tg.Matcher = &elbv2.Matcher{HttpCode: aws.String(r.HealthCheck.Matcher)}
 	}
 
 	// Omitted for NLB (TCP/UDP/TLS) target groups, matching AWS, which returns

spinifex/domains/elbv2/service_impl_rules.go

@@ -591,8 +591,8 @@ func (s *ELBv2ServiceImpl) validateAndConvertRuleActions(ctx context.Context, in
 				return nil, err
 			}
 		case ActionTypeFixedResponse:
-			if action.FixedResponse == nil {
-				return nil, errors.New(awserrors.ErrorMissingParameter)
+			if err := validateFixedResponseAction(action.FixedResponse); err != nil {
+				return nil, err
 			}
 		default:
 			return nil, errors.New(awserrors.ErrorELBv2InvalidConfigurationRequest)

spinifex/domains/elbv2/service_impl_rules_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/elbv2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -660,3 +661,53 @@ func sortByPriority(rs []*RuleRecord) []*RuleRecord {
 	}
 	return out
 }
+
+func TestValidateListenerAction_FixedResponse(t *testing.T) {
+	t.Parallel()
+	fixed := func(ct, body string) ListenerAction {
+		return ListenerAction{Type: ActionTypeFixedResponse, FixedResponse: &FixedResponseAction{
+			StatusCode: "200", ContentType: ct, MessageBody: body,
+		}}
+	}
+
+	for _, ct := range []string{"", "text/plain", "text/css", "text/html", "application/javascript", "application/json"} {
+		assert.NoError(t, validateListenerAction(fixed(ct, "ok")), ct)
+	}
+	assert.NoError(t, validateListenerAction(fixed("text/plain", "")))
+	assert.NoError(t, validateListenerAction(fixed("text/plain", "sk=${LB_SECRET_KEY} it's \"q\" #x")))
+	assert.NoError(t, validateListenerAction(fixed("text/plain", strings.Repeat("é", 1024))))
+
+	bad := []ListenerAction{
+		fixed("text/$LB_SECRET_KEY", "ok"),
+		fixed("text/xml", "ok"),
+		fixed("TEXT/PLAIN", "ok"),
+		fixed("text/plain", strings.Repeat("a", 1025)),
+		fixed("text/plain", "line\nacl x always_true"),
+		fixed("text/plain", "cr\r"),
+		fixed("text/plain", "nul\x00"),
+	}
+	for _, a := range bad {
+		assert.EqualError(t, validateListenerAction(a), awserrors.ErrorInvalidParameterValue, "%+v", *a.FixedResponse)
+	}
+	assert.EqualError(t, validateListenerAction(ListenerAction{Type: ActionTypeFixedResponse}), awserrors.ErrorMissingParameter)
+}
+
+func TestCreateRule_RejectsFixedResponseContentType(t *testing.T) {
+	t.Parallel()
+	env := newRuleTestEnv(t, "cr-badfixed")
+
+	_, err := env.svc.CreateRule(context.Background(), &elbv2.CreateRuleInput{
+		ListenerArn: aws.String(env.listenerArn),
+		Priority:    aws.Int64(1),
+		Conditions:  []*elbv2.RuleCondition{{Field: aws.String("path-pattern"), Values: aws.StringSlice([]string{"/x"})}},
+		Actions: []*elbv2.Action{{
+			Type: aws.String("fixed-response"),
+			FixedResponseConfig: &elbv2.FixedResponseActionConfig{
+				StatusCode:  aws.String("200"),
+				ContentType: aws.String("text/$LB_SECRET_KEY"),
+				MessageBody: aws.String("ok"),
+			},
+		}},
+	}, testAccountID)
+	require.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+}

spinifex/domains/elbv2/service_impl_test.go

@@ -459,11 +459,12 @@ func TestCreateTargetGroup_TCPProtocol(t *testing.T) {
 	assert.Equal(t, int64(5432), *tg.Port)
 	// NLB health check defaults: TCP protocol, no path, no matcher.
 	assert.Equal(t, "TCP", *tg.HealthCheckProtocol)
-	assert.Empty(t, *tg.HealthCheckPath)
-	assert.Empty(t, *tg.Matcher.HttpCode)
+	assert.Nil(t, tg.HealthCheckPath)
+	assert.Nil(t, tg.Matcher)
+	assert.Equal(t, int64(30), *tg.HealthCheckIntervalSeconds)
 	assert.Equal(t, int64(10), *tg.HealthCheckTimeoutSeconds)
-	assert.Equal(t, int64(3), *tg.HealthyThresholdCount)
-	assert.Equal(t, int64(3), *tg.UnhealthyThresholdCount)
+	assert.Equal(t, int64(5), *tg.HealthyThresholdCount)
+	assert.Equal(t, int64(2), *tg.UnhealthyThresholdCount)
 }
 
 func TestCreateTargetGroup_NLBProtocols(t *testing.T) {

spinifex/domains/elbv2/types.go

@@ -99,8 +99,8 @@ const (
 	// Default health check values (NLB).
 	DefaultNLBHealthCheckInterval = 30
 	DefaultNLBHealthCheckTimeout  = 10
-	DefaultNLBHealthyThreshold    = 3
-	DefaultNLBUnhealthyThreshold  = 3
+	DefaultNLBHealthyThreshold    = 5
+	DefaultNLBUnhealthyThreshold  = 2
 	DefaultNLBHealthCheckProtocol = ProtocolTCP
 	DefaultNLBHealthCheckPort     = "traffic-port"
 

spinifex/domains/iam/account_generation_test.go

@@ -33,9 +33,15 @@ func TestSetAccountStatusCountsOnlyChanges(t *testing.T) {
 		{AccountStatusTerminating, 3},
 	}
 	for _, step := range steps {
-		got, err := svc.SetAccountStatus(account.AccountID, step.status)
-		require.NoError(t, err)
-		assert.Equal(t, step.want, got.Generation, "status %s", step.status)
+		if step.status == AccountStatusTerminating {
+			generation, err := svc.QuiesceAccount(account.AccountID, "op-1", 1)
+			require.NoError(t, err)
+			assert.Equal(t, step.want, generation, "status %s", step.status)
+		} else {
+			got, err := svc.SetAccountStatus(account.AccountID, step.status)
+			require.NoError(t, err)
+			assert.Equal(t, step.want, got.Generation, "status %s", step.status)
+		}
 
 		stored, err := svc.GetAccount(account.AccountID)
 		require.NoError(t, err)