test(uefi): separate the embedded store from the injected error in the failing-get fake

A launch whose record points at a variable store keeps the live copy at
<DataDir>/uefi/<instance-id>/vars.fd and opens it as pflash unit 1 with
format=raw. It reuses the copy when seeded.json beside it names the
pointer, and otherwise gets the object, decompresses it, checks it is
exactly the node's VARS size and writes vars.fd then seeded.json by
temporary file and rename. A failed get is retryable like a mount; a
missing or malformed object is not.

A store whose firmware arch or VARS size differs from the node's fails
the launch before any volume is mounted, with a Server.InternalError
state reason and a log naming both firmwares. A different CODE hash
boots, and is logged and counted.

A UEFI instance with no pointer and no EFI volume, or with both, is
refused with InternalError before its record is changed, in launch,
Start and the claimed-instance start. An instance whose variables are on
an EFI volume still launches through it.

.github/workflows/e2e-baremetal.yml

@@ -71,9 +71,16 @@ jobs:
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ inputs.cell || '20' }}-${{ github.run_id }}
       SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs || format('ci.run_id={0},ci.workflow={1},ci.cell={2},ci.matrix=baremetal/pxe/real-hw', github.run_id, github.workflow, inputs.cell || '20') }}
     steps:
+      # A cancelled job's root steps outlive it, since the runner cannot reach
+      # them once it kills sudo, and would keep writing here and to the cache.
+      # A rootfs killed before bin/bash exists is one the build refuses to reuse.
       - name: Clean Workspace
         run: |
+          sudo systemctl stop spinifex-bm-pxe.scope spinifex-bm-e2e.scope 2>/dev/null || true
           sudo umount -R -l "${BM_ISO_CACHE}/rootfs" 2>/dev/null || true
+          if [ -d "${BM_ISO_CACHE}/rootfs" ] && [ ! -x "${BM_ISO_CACHE}/rootfs/bin/bash" ]; then
+            sudo rm -rf --one-file-system "${BM_ISO_CACHE}/rootfs"
+          fi
           if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
           sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
@@ -113,10 +120,15 @@ jobs:
             "$GITHUB_WORKSPACE" \
             "${RUNNER_TEMP}/spinifex-tests.tar" \
             "${RUNNER_TEMP}/bm-bins"
+          # Built here, as the runner user, so the PXE build finds it current.
+          make -C "$GITHUB_WORKSPACE" build-microvm-image
 
       - name: Build PXE Artifacts
         run: |
-          sudo make -C mulga/scripts/iso-builder \
+          # -o build-spinifex: the binaries come from Build Artifacts, and a
+          # rebuild here would write root-owned files into the workspace.
+          sudo systemd-run --scope --collect --quiet --unit=spinifex-bm-pxe -- \
+            make -C mulga/scripts/iso-builder -o build-spinifex \
             build-rootfs build-squashfs build-initramfs \
             CACHE="${BM_ISO_CACHE}" \
             SPINIFEX_BIN="${RUNNER_TEMP}/bm-bins/spx" \
@@ -124,6 +136,10 @@ jobs:
             SPINIFEX_SRC="${GITHUB_WORKSPACE}" \
             VIPERBLOCK_SRC="${RUNNER_TEMP}/bm-bins/viperblock" \
             VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always --exclude '*-dev' 2>/dev/null || echo dev)"
+          if root_owned="$(find "$GITHUB_WORKSPACE" -user root -print -quit)" && [ -n "$root_owned" ]; then
+            echo "::error::the PXE build wrote root-owned files into the workspace, e.g. $root_owned"
+            exit 1
+          fi
 
       - name: Write ePDU credentials
         run: |
@@ -195,13 +211,18 @@ jobs:
           done
           export BM_TEST_TIMEOUT
           echo "suites=${BM_SUITES} test-timeout=${BM_TEST_TIMEOUT}"
-          sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
+          sudo -E systemd-run --scope --collect --quiet --unit=spinifex-bm-e2e -- \
+            mulga/scripts/baremetal/run-baremetal-e2e.sh \
             --vmlinuz   "${BM_ISO_CACHE}/vmlinuz" \
             --initrd    "${BM_ISO_CACHE}/initrd.img" \
             --squashfs  "${BM_ISO_CACHE}/spinifex.squashfs" \
             --tests-tar "${RUNNER_TEMP}/spinifex-tests.tar" \
             2>&1 | tee "${ARTIFACT_DIR}/test-baremetal.log"
 
+      - name: Stop this job's root steps
+        if: always()
+        run: sudo systemctl stop spinifex-bm-pxe.scope spinifex-bm-e2e.scope 2>/dev/null || true
+
       - name: Generate JUnit report
         if: always()
         run: |

architecture/layering/allowlist_test.go

@@ -15,9 +15,6 @@ var allowlist = []allowance{
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/igw", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/image", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/instance", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
-	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/key", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
-	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/launchtemplate", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
-	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/placementgroup", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/routetable", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/snapshot", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/volume", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},

cmd/lambda-agent/package_testlink.go

@@ -0,0 +1,276 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// Dev-only code delivery for the test path: the host attaches the ZIP as a
+// read-only disk and passes its exact size and SHA-256 through fw_cfg. This
+// is not the production delivery, and these bounds are not AWS limits.
+const (
+	entryPackageSize   = "test-package-size"
+	entryPackageSHA256 = "test-package-sha256"
+
+	devMaxZipBytes    = 16 << 20
+	devMaxUnzipBytes  = 64 << 20
+	devMaxEntries     = 1024
+	packageDevice     = "/dev/vda"
+	taskTmpfsSizeOpts = "mode=0755,size=72m"
+)
+
+type packageSpec struct {
+	size   int64
+	sha256 [sha256.Size]byte
+}
+
+type unpackLimits struct {
+	entries int
+	bytes   int64
+}
+
+var devLimits = unpackLimits{entries: devMaxEntries, bytes: devMaxUnzipBytes}
+
+// readPackageSpec reads the expected size and SHA-256 from fw_cfg; it must
+// run before fw_cfg is sealed.
+func readPackageSpec(fwCfgDir string) (packageSpec, error) {
+	var spec packageSpec
+	read := func(name string) (string, error) {
+		b, err := os.ReadFile(filepath.Join(fwCfgDir, name, "raw"))
+		if err != nil {
+			return "", fmt.Errorf("read %s: %w", name, err)
+		}
+		return strings.TrimSpace(string(b)), nil
+	}
+	s, err := read(entryPackageSize)
+	if err != nil {
+		return spec, err
+	}
+	spec.size, err = strconv.ParseInt(s, 10, 64)
+	if err != nil || spec.size <= 0 || spec.size > devMaxZipBytes {
+		return spec, fmt.Errorf("package size %q outside 1..%d bytes", s, devMaxZipBytes)
+	}
+	h, err := read(entryPackageSHA256)
+	if err != nil {
+		return spec, err
+	}
+	b, err := hex.DecodeString(h)
+	if err != nil || len(b) != sha256.Size {
+		return spec, errors.New("package sha256 is not 64 hex digits")
+	}
+	copy(spec.sha256[:], b)
+	return spec, nil
+}
+
+// taskPackage puts the package under dir: a fresh nosuid,nodev tmpfs, made
+// read-only once the files are in place.
+type taskPackage struct {
+	device     string
+	dir        string
+	loadDriver func() error
+	mountDir   func(dir string) error
+	sealDir    func(dir string) error
+}
+
+func devTaskPackage() taskPackage {
+	return taskPackage{device: packageDevice, dir: taskRoot, loadDriver: loadBlockDriver, mountDir: mountTaskDir, sealDir: sealTaskDir}
+}
+
+func (p taskPackage) install(spec packageSpec) error {
+	if err := p.loadDriver(); err != nil {
+		return fmt.Errorf("load block driver: %w", err)
+	}
+	dev, err := os.Open(p.device)
+	if err != nil {
+		return fmt.Errorf("open package disk: %w", err)
+	}
+	defer dev.Close()
+	if err := p.mountDir(p.dir); err != nil {
+		return fmt.Errorf("mount %s: %w", p.dir, err)
+	}
+	if err := unpack(dev, spec, p.dir, devLimits); err != nil {
+		return err
+	}
+	if err := p.sealDir(p.dir); err != nil {
+		return fmt.Errorf("make %s read-only: %w", p.dir, err)
+	}
+	return nil
+}
+
+// unpack reads exactly spec.size bytes once, checks their SHA-256, then
+// validates every entry before writing any of them under dir.
+func unpack(r io.ReaderAt, spec packageSpec, dir string, lim unpackLimits) error {
+	buf := make([]byte, spec.size)
+	if _, err := io.ReadFull(io.NewSectionReader(r, 0, spec.size), buf); err != nil {
+		return fmt.Errorf("read package: %w", err)
+	}
+	if sha256.Sum256(buf) != spec.sha256 {
+		return errors.New("package sha256 mismatch")
+	}
+	zr, err := zip.NewReader(bytes.NewReader(buf), spec.size)
+	if err != nil {
+		return fmt.Errorf("open package: %w", err)
+	}
+	if len(zr.File) > lim.entries {
+		return fmt.Errorf("package has %d entries, limit %d", len(zr.File), lim.entries)
+	}
+	names := make([]string, len(zr.File))
+	var declared int64
+	for i, f := range zr.File {
+		name, err := entryName(f.Name)
+		if err != nil {
+			return err
+		}
+		names[i] = name
+		switch mode := f.Mode(); {
+		case mode.IsDir():
+		case mode.IsRegular():
+			// Compared against what remains, so a huge size cannot wrap the sum.
+			if f.UncompressedSize64 > uint64(lim.bytes-declared) {
+				return fmt.Errorf("%q: unpacked size over the %d-byte limit", f.Name, lim.bytes)
+			}
+			declared += int64(f.UncompressedSize64)
+		default:
+			return fmt.Errorf("%q: %s entries are refused", f.Name, mode.Type())
+		}
+	}
+
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		return err
+	}
+	defer root.Close()
+	var written int64
+	for i, f := range zr.File {
+		n, err := extract(root, names[i], f, lim.bytes-written)
+		written += n
+		if err != nil {
+			return fmt.Errorf("%q: %w", f.Name, err)
+		}
+	}
+	return nil
+}
+
+// extract writes one entry, refusing it if its actual bytes exceed budget
+// whatever its header claims.
+func extract(root *os.Root, name string, f *zip.File, budget int64) (int64, error) {
+	if f.Mode().IsDir() {
+		return 0, mkdirs(root, name)
+	}
+	if d := path.Dir(name); d != "." {
+		if err := mkdirs(root, d); err != nil {
+			return 0, err
+		}
+	}
+	perm := os.FileMode(0o644)
+	if f.Mode()&0o111 != 0 {
+		perm = 0o755
+	}
+	in, err := f.Open()
+	if err != nil {
+		return 0, err
+	}
+	defer in.Close()
+	out, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
+	if err != nil {
+		return 0, err
+	}
+	// The umask applies at create; fchmod does not, so the mode is exact.
+	if err := out.Chmod(perm); err != nil {
+		out.Close()
+		return 0, err
+	}
+	n, err := io.Copy(out, io.LimitReader(in, budget+1))
+	if cerr := out.Close(); err == nil {
+		err = cerr
+	}
+	if err == nil && n > budget {
+		err = errors.New("unpacked bytes over the limit")
+	}
+	return n, err
+}
+
+// mkdirs creates each component of dir as mode 0755 whatever the umask. An
+// existing component must be a real directory, never a symlink.
+func mkdirs(root *os.Root, dir string) error {
+	cur := ""
+	for _, part := range strings.Split(dir, "/") {
+		cur = path.Join(cur, part)
+		if err := root.Mkdir(cur, 0o755); err != nil {
+			if !errors.Is(err, fs.ErrExist) {
+				return err
+			}
+			fi, err := root.Lstat(cur)
+			if err != nil {
+				return err
+			}
+			if !fi.IsDir() {
+				return fmt.Errorf("%q exists and is not a directory", cur)
+			}
+		}
+		if err := root.Chmod(cur, 0o755); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// entryName returns a clean relative path, refusing anything that could name
+// a location outside the task root.
+func entryName(name string) (string, error) {
+	if name == "" || strings.ContainsAny(name, "\\\x00") || path.IsAbs(name) {
+		return "", fmt.Errorf("%q: entry name refused", name)
+	}
+	c := path.Clean(name)
+	if c == "." || c == ".." || strings.HasPrefix(c, "../") {
+		return "", fmt.Errorf("%q: entry name refused", name)
+	}
+	return c, nil
+}
+
+// loadBlockDriver loads virtio_blk, which the image ships but init does not
+// load, then waits briefly for the package disk to appear.
+func loadBlockDriver() error {
+	if err := exec.Command("/sbin/modprobe", "virtio_blk").Run(); err != nil {
+		var uts unix.Utsname
+		if uerr := unix.Uname(&uts); uerr != nil {
+			return uerr
+		}
+		ko := "/lib/modules/" + unix.ByteSliceToString(uts.Release[:]) + "/kernel/drivers/block/virtio_blk.ko"
+		if err := exec.Command("/sbin/insmod", ko).Run(); err != nil {
+			return fmt.Errorf("modprobe and insmod virtio_blk failed: %w", err)
+		}
+	}
+	for range 50 {
+		if _, err := os.Stat(packageDevice); err == nil {
+			return nil
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	return fmt.Errorf("%s did not appear", packageDevice)
+}
+
+func mountTaskDir(dir string) error {
+	return unix.Mount("tmpfs", dir, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, taskTmpfsSizeOpts)
+}
+
+func sealTaskDir(dir string) error {
+	return unix.Mount("", dir, "", unix.MS_REMOUNT|unix.MS_RDONLY|unix.MS_NOSUID|unix.MS_NODEV, "")
+}

cmd/lambda-agent/package_testlink_test.go

@@ -0,0 +1,209 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"hash/crc32"
+	"io/fs"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testZip builds a ZIP; an entry named bootstrap is marked executable.
+func testZip(t *testing.T, files map[string]string) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	names := make([]string, 0, len(files))
+	for n := range files {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	for _, n := range names {
+		h := &zip.FileHeader{Name: n, Method: zip.Deflate}
+		h.SetMode(0o644)
+		if n == "bootstrap" {
+			h.SetMode(0o755)
+		}
+		w, err := zw.CreateHeader(h)
+		require.NoError(t, err)
+		_, err = w.Write([]byte(files[n]))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+// rawZip writes entries with headers set by the caller, so a test can claim
+// sizes or modes the data does not have.
+func rawZip(t *testing.T, entries ...*zip.FileHeader) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for _, h := range entries {
+		w, err := zw.CreateRaw(h)
+		require.NoError(t, err)
+		_, err = w.Write([]byte("x"))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+func specFor(b []byte) packageSpec {
+	return packageSpec{size: int64(len(b)), sha256: sha256.Sum256(b)}
+}
+
+// Not parallel: the umask is process-wide, so the test sets a restrictive one
+// and restores it.
+func TestUnpackInstallsFilesWithFixedModes(t *testing.T) {
+	old := syscall.Umask(0o077)
+	t.Cleanup(func() { syscall.Umask(old) })
+	b := testZip(t, map[string]string{"bootstrap": "#!/bin/sh\n", "lib/data.txt": "data"})
+	dir := t.TempDir()
+	require.NoError(t, unpack(bytes.NewReader(b), specFor(b), dir, devLimits))
+
+	pfi, err := os.Stat(filepath.Join(dir, "lib"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o755), pfi.Mode().Perm(), "implicit parent dir")
+
+	fi, err := os.Stat(filepath.Join(dir, "bootstrap"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o755), fi.Mode().Perm(), "executable")
+	fi, err = os.Stat(filepath.Join(dir, "lib", "data.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o644), fi.Mode().Perm())
+	got, err := os.ReadFile(filepath.Join(dir, "lib", "data.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, "data", string(got))
+}
+
+func TestUnpackRefusesBeforeWritingAnything(t *testing.T) {
+	symlink := &zip.FileHeader{Name: "link", Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: 1}
+	symlink.SetMode(fs.ModeSymlink | 0o777)
+	fifo := &zip.FileHeader{Name: "pipe", Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: 1}
+	fifo.SetMode(fs.ModeNamedPipe | 0o644)
+	// The second size would wrap a plain running sum back under the limit.
+	big := func(name string, n uint64) *zip.FileHeader {
+		return &zip.FileHeader{Name: name, Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: n}
+	}
+	cases := map[string]struct {
+		zip  []byte
+		want string
+	}{
+		"parent traversal": {testZip(t, map[string]string{"a": "ok", "../escape": "x"}), "entry name refused"},
+		"nested traversal": {testZip(t, map[string]string{"a": "ok", "lib/../../escape": "x"}), "entry name refused"},
+		"absolute path":    {testZip(t, map[string]string{"a": "ok", "/etc/escape": "x"}), "entry name refused"},
+		"backslash":        {testZip(t, map[string]string{"a": "ok", `..\escape`: "x"}), "entry name refused"},
+		"symlink":          {rawZip(t, big("a", 1), symlink), "entries are refused"},
+		"named pipe":       {rawZip(t, big("a", 1), fifo), "entries are refused"},
+		"declared size":    {rawZip(t, big("a", devMaxUnzipBytes+1)), "over the"},
+		"wrapping sizes":   {rawZip(t, big("a", devMaxUnzipBytes), big("b", math.MaxUint64)), "over the"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := t.TempDir()
+			err := unpack(bytes.NewReader(tc.zip), specFor(tc.zip), dir, devLimits)
+			assert.ErrorContains(t, err, tc.want)
+			left, rerr := os.ReadDir(dir)
+			require.NoError(t, rerr)
+			assert.Empty(t, left, "files written before the refusal")
+		})
+	}
+}
+
+func TestUnpackRefusesTooManyEntries(t *testing.T) {
+	b := testZip(t, map[string]string{"a": "1", "b": "2", "c": "3"})
+	err := unpack(bytes.NewReader(b), specFor(b), t.TempDir(), unpackLimits{entries: 2, bytes: devMaxUnzipBytes})
+	assert.ErrorContains(t, err, "limit 2")
+}
+
+// The header claims a size within the limit, but the data is longer: the
+// actual bytes are refused, whatever the header says.
+func TestUnpackRefusesBytesBeyondTheLimit(t *testing.T) {
+	data := bytes.Repeat([]byte("a"), 64)
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	h := &zip.FileHeader{Name: "a", Method: zip.Store, CRC32: crc32.ChecksumIEEE(data), CompressedSize64: uint64(len(data)), UncompressedSize64: 8}
+	w, err := zw.CreateRaw(h)
+	require.NoError(t, err)
+	_, err = w.Write(data)
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	b := buf.Bytes()
+
+	dir := t.TempDir()
+	err = unpack(bytes.NewReader(b), specFor(b), dir, unpackLimits{entries: 4, bytes: 16})
+	require.Error(t, err)
+	fi, serr := os.Stat(filepath.Join(dir, "a"))
+	if serr == nil {
+		assert.LessOrEqual(t, fi.Size(), int64(17))
+	}
+}
+
+func TestUnpackChecksSizeAndHash(t *testing.T) {
+	b := testZip(t, map[string]string{"bootstrap": "x"})
+	wrongHash := specFor(b)
+	wrongHash.sha256[0] ^= 1
+	assert.ErrorContains(t, unpack(bytes.NewReader(b), wrongHash, t.TempDir(), devLimits), "sha256 mismatch")
+
+	longer := specFor(b)
+	longer.size++
+	assert.ErrorContains(t, unpack(bytes.NewReader(b), longer, t.TempDir(), devLimits), "read package")
+}
+
+func TestReadPackageSpecBounds(t *testing.T) {
+	sum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	cases := map[string]struct {
+		size, hash string
+		want       string
+	}{
+		"valid":     {"10", sum, ""},
+		"zero":      {"0", sum, "outside"},
+		"too large": {"16777217", sum, "outside"},
+		"not hex":   {"10", "zz", "64 hex digits"},
+		"short":     {"10", sum[:62], "64 hex digits"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := t.TempDir()
+			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash} {
+				require.NoError(t, os.MkdirAll(filepath.Join(dir, n), 0o700))
+				require.NoError(t, os.WriteFile(filepath.Join(dir, n, "raw"), []byte(v), 0o600))
+			}
+			spec, err := readPackageSpec(dir)
+			if tc.want == "" {
+				require.NoError(t, err)
+				assert.Equal(t, int64(10), spec.size)
+				return
+			}
+			assert.ErrorContains(t, err, tc.want)
+		})
+	}
+}
+
+// A pre-existing symlink where a directory belongs is refused, not followed
+// or chmodded.
+func TestMkdirsRefusesSymlinkComponent(t *testing.T) {
+	dir := t.TempDir()
+	outside := t.TempDir()
+	require.NoError(t, os.Chmod(outside, 0o700))
+	require.NoError(t, os.Symlink(outside, filepath.Join(dir, "lib")))
+	root, err := os.OpenRoot(dir)
+	require.NoError(t, err)
+	defer root.Close()
+	assert.ErrorContains(t, mkdirs(root, "lib/sub"), "not a directory")
+	fi, err := os.Stat(outside)
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o700), fi.Mode().Perm(), "symlink target was chmodded")
+}

cmd/lambda-agent/start_testlink.go

@@ -30,6 +30,7 @@ var handler = flag.String("handler", os.Getenv("_HANDLER"), "function handler pa
 // connection; the carrier is not the guest protocol.
 type testlink struct {
 	startup
+	pkg            taskPackage
 	runtimeAddr    string
 	startBootstrap func(ctx context.Context, env []string) (*exec.Cmd, error)
 }
@@ -37,6 +38,7 @@ type testlink struct {
 func start(ctx context.Context) int {
 	t := testlink{
 		startup:     defaultStartup(),
+		pkg:         devTaskPackage(),
 		runtimeAddr: runtimeAddr,
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			return agent.StartBootstrap(ctx, taskRoot+"/bootstrap", env, os.Stdout, os.Stderr)
@@ -50,11 +52,16 @@ func start(ctx context.Context) int {
 }
 
 func (t testlink) run(parent context.Context) (int, error) {
+	// The package values are fw_cfg entries, so read them before connect seals it.
+	spec, specErr := readPackageSpec(t.fwCfgDir)
 	conn, code, err := t.connect(parent)
 	if err != nil {
-		return code, err
+		return code, errors.Join(err, specErr)
 	}
 	defer conn.Close()
+	if specErr != nil {
+		return exitRefused, specErr
+	}
 	link := hostlink.NewTestStream(conn)
 
 	// The handshake can return before the host refuses the guest, so nothing
@@ -69,6 +76,9 @@ func (t testlink) run(parent context.Context) (int, error) {
 	if parent.Err() != nil {
 		return exitLink, context.Cause(parent)
 	}
+	if err := t.pkg.install(spec); err != nil {
+		return exitRefused, fmt.Errorf("install package: %w", err)
+	}
 
 	srv := runtimeapi.New()
 	ln, err := net.Listen("tcp", t.runtimeAddr)

cmd/lambda-agent/start_testlink_test.go

@@ -5,10 +5,14 @@ package main
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"io"
 	"net/http"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"strconv"
 	"testing"
 	"time"
 
@@ -43,12 +47,31 @@ func TestHelperBootstrap(t *testing.T) {
 type bootstrapStarts struct {
 	n       int
 	started chan struct{}
+	task    string
+	sealed  bool
 }
 
-func (b *bootstrapStarts) testlink(g *guest) testlink {
+// testlink builds a guest whose fw_cfg also carries a controlled package,
+// installed into a temp dir without mounting anything.
+func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) testlink {
+	t.Helper()
+	zipBytes := testZip(t, map[string]string{"bootstrap": "#!/bin/sh\n", "lib/data.txt": "data"})
+	disk := filepath.Join(t.TempDir(), "disk")
+	require.NoError(t, os.WriteFile(disk, zipBytes, 0o600))
+	sum := sha256.Sum256(zipBytes)
+	entries[entryPackageSize] = []byte(strconv.Itoa(len(zipBytes)))
+	entries[entryPackageSHA256] = []byte(hex.EncodeToString(sum[:]))
+	g := newGuest(t, entries)
 	b.started = make(chan struct{}, 1)
+	b.task = t.TempDir()
 	return testlink{
-		startup:     g.startup(),
+		startup: g.startup(),
+		pkg: taskPackage{
+			device: disk, dir: b.task,
+			loadDriver: func() error { return nil },
+			mountDir:   func(string) error { return nil },
+			sealDir:    func(string) error { b.sealed = true; return nil },
+		},
 		runtimeAddr: "127.0.0.1:0",
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			b.n++
@@ -80,7 +103,7 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
 
 	conn := h.guest(t)
 	defer conn.Close()
@@ -109,6 +132,8 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		assert.Error(t, err, "agent answered a second invocation")
 	}
 	assert.Equal(t, 1, b.n)
+	assert.True(t, b.sealed, "task dir not made read-only")
+	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
 }
 
 func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
@@ -117,7 +142,7 @@ func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
 
 	conn := h.guest(t)
 	defer conn.Close()
@@ -142,7 +167,7 @@ func TestTestlinkHostRefusalNeverStartsBootstrap(t *testing.T) {
 	entries[lambdav1.EntryGuestKey], entries[lambdav1.EntryGuestCert] = wrong[lambdav1.EntryGuestKey], wrong[lambdav1.EntryGuestCert]
 	var b bootstrapStarts
 
-	res := wait(t, runTestlinkAsync(ctx, b.testlink(newGuest(t, entries))), "agent to stop")
+	res := wait(t, runTestlinkAsync(ctx, b.testlink(t, entries)), "agent to stop")
 	assert.Equal(t, exitLink, res.code)
 	require.Error(t, res.err)
 	assert.Contains(t, res.err.Error(), "bad certificate")
@@ -155,7 +180,7 @@ func TestTestlinkHostCloseBeforeFrameNeverStartsBootstrap(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
 
 	require.NoError(t, h.guest(t).Close())
 	res := wait(t, done, "agent to stop")
@@ -177,3 +202,45 @@ func TestOneInvocationHonoursCancelledContext(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "r-1", req.RequestID)
 }
+
+// A package that fails its check is refused after authentication, and the
+// bootstrap never starts.
+func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	tl := b.testlink(t, h.entries())
+	tl.pkg.device = filepath.Join(t.TempDir(), "other")
+	require.NoError(t, os.WriteFile(tl.pkg.device, testZip(t, map[string]string{"bootstrap": "x"}), 0o600))
+	done := runTestlinkAsync(ctx, tl)
+
+	conn := h.guest(t)
+	defer conn.Close()
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{RequestID: "r-1"}))
+	res := wait(t, done, "agent to stop")
+	assert.Equal(t, exitRefused, res.code)
+	assert.ErrorContains(t, res.err, "install package")
+	assert.Zero(t, b.n)
+	assert.False(t, b.sealed)
+}
+
+// Missing package values still seal fw_cfg, and the agent refuses before
+// waiting for any frame.
+func TestTestlinkMissingPackageSpecRefuses(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	tl := b.testlink(t, h.entries())
+	require.NoError(t, os.RemoveAll(filepath.Join(tl.fwCfgDir, entryPackageSHA256)))
+
+	res := wait(t, runTestlinkAsync(ctx, tl), "agent to stop")
+	assert.Equal(t, exitRefused, res.code)
+	assert.ErrorContains(t, res.err, "read "+entryPackageSHA256)
+	assert.Zero(t, b.n)
+	_, err := os.Stat(tl.fwCfgRoot)
+	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg not sealed")
+}

contracts/operator/v1/admin.go

@@ -25,6 +25,10 @@ const (
 	DeletionStateRunning   = "RUNNING"
 	DeletionStateCompleted = "COMPLETED"
 	DeletionStateFailed    = "FAILED"
+
+	// DeletionStateExercised ends a non-finalizing run: the account is left
+	// TERMINATING and a later DeleteAccount takes the same operation over.
+	DeletionStateExercised = "EXERCISED"
 )
 
 // CreateAccountRequest is the /admin/CreateAccount request body.
@@ -52,6 +56,10 @@ type DeleteAccountRequest struct {
 	ClientToken string `json:"clientToken"`
 	Force       bool   `json:"force,omitempty"`
 	DryRun      bool   `json:"dryRun,omitempty"`
+
+	// NonFinalizing quiesces the account, cleans the quiesced EC2 platform
+	// kinds and stops. No legacy stage runs and the account record stays.
+	NonFinalizing bool `json:"nonFinalizing,omitempty"`
 }
 
 // DescribeAccountDeletionRequest is the /admin/DescribeAccountDeletion body.
@@ -76,8 +84,8 @@ type ListAccountsResponse struct {
 }
 
 // IAMQuiescence is the coordinator's recorded view of IAM quiescence for one
-// fence. Recording only: it admits nothing, and QuiescedScope stays empty until
-// a final inventory re-read exists to catch a late write.
+// fence. Recording only: it admits nothing. QuiescedScope is filled only by a run
+// that guarantees a final inventory re-read, and holds as of EvaluatedAt.
 type IAMQuiescence struct {
 	OperationID   string            `json:"operationId"`
 	Fence         uint64            `json:"fence"`
@@ -136,3 +144,67 @@ const (
 	ReasonOwnerInventoryIncomplete     = "owner-inventory-incomplete"
 	ReasonFinalInventoryRereadNotWired = "final-inventory-reread-not-wired"
 )
+
+// ExerciseResult is what a non-finalizing run did after IAM quiescence.
+type ExerciseResult struct {
+	// Skipped is the fixed reason no cleanup ran; Cleanup is then absent.
+	Skipped    string           `json:"skipped,omitempty"`
+	Cleanup    *PlatformCleanup `json:"cleanup,omitempty"`
+	Accounting []KindAccounting `json:"accounting,omitempty"`
+}
+
+// PlatformCleanup is the EC2 platform cleanup result. It carries resource IDs,
+// fixed reasons and counts only, never names, tags, key material or data.
+type PlatformCleanup struct {
+	Kinds  []CleanupKind  `json:"kinds"`
+	Items  []CleanupItem  `json:"items,omitempty"`
+	Counts []CleanupCount `json:"counts,omitempty"`
+}
+
+// CleanupKind is one kind's scan status after the cleanup's own re-scan.
+type CleanupKind struct {
+	Kind              string   `json:"kind"`
+	Status            string   `json:"status"`
+	IncompleteReasons []string `json:"incompleteReasons,omitempty"`
+}
+
+// CleanupItem is one resource by its ID with what the cleanup did to it.
+type CleanupItem struct {
+	ResourceID  string `json:"resourceId"`
+	Kind        string `json:"kind"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	MemberCount int    `json:"memberCount,omitempty"`
+}
+
+// CleanupCount is the number of records of one class that have no public ID.
+type CleanupCount struct {
+	Kind        string `json:"kind"`
+	Class       string `json:"class"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	Count       int    `json:"count"`
+}
+
+// KindAccounting is the final inventory re-read held against the cleanup result
+// for one kind. Quiescence is not rewritten from it.
+type KindAccounting struct {
+	Kind      string `json:"kind"`
+	Status    string `json:"status"`
+	Reason    string `json:"reason,omitempty"`
+	Remaining int    `json:"remaining"`
+}
+
+// Accounting statuses and fixed reasons of a KindAccounting.
+const (
+	AccountingAccounted   = "accounted"
+	AccountingUnaccounted = "unaccounted"
+
+	AccountingReasonRereadIncomplete = "reread-incomplete"
+	AccountingReasonUnnamedRemainder = "unnamed-remainder"
+)
+
+// Fixed reasons an exercise ran no cleanup.
+const (
+	ExerciseSkippedNoQuiescedKind = "no-quiesced-kind"
+)

contracts/operator/v1/admin_test.go

@@ -43,6 +43,27 @@ func TestAdminWire_DeletionStates(t *testing.T) {
 	require.Equal(t, "RUNNING", DeletionStateRunning)
 	require.Equal(t, "COMPLETED", DeletionStateCompleted)
 	require.Equal(t, "FAILED", DeletionStateFailed)
+	require.Equal(t, "EXERCISED", DeletionStateExercised)
+}
+
+func TestAdminWire_NonFinalizingDeleteRequest(t *testing.T) {
+	goldenJSON(t, DeleteAccountRequest{AccountID: "000000000042", AccountName: "acme", ClientToken: "tok-3", NonFinalizing: true},
+		`{"accountId":"000000000042","accountName":"acme","clientToken":"tok-3","nonFinalizing":true}`)
+}
+
+func TestAdminWire_ExerciseResult(t *testing.T) {
+	goldenJSON(t, ExerciseResult{
+		Cleanup: &PlatformCleanup{
+			Kinds:  []CleanupKind{{Kind: "key-pair", Status: "observed-clean-at-scan-time"}},
+			Items:  []CleanupItem{{ResourceID: "pg-1", Kind: "placement-group", Disposition: "blocked", Reason: "instance-members", MemberCount: 1}},
+			Counts: []CleanupCount{{Kind: "key-pair", Class: "public-key-object-without-metadata", Disposition: "deleted", Count: 2}},
+		},
+		Accounting: []KindAccounting{{Kind: "placement-group", Status: AccountingAccounted, Remaining: 1}},
+	}, `{"cleanup":{"kinds":[{"kind":"key-pair","status":"observed-clean-at-scan-time"}],
+		"items":[{"resourceId":"pg-1","kind":"placement-group","disposition":"blocked","reason":"instance-members","memberCount":1}],
+		"counts":[{"kind":"key-pair","class":"public-key-object-without-metadata","disposition":"deleted","count":2}]},
+		"accounting":[{"kind":"placement-group","status":"accounted","remaining":1}]}`)
+	goldenJSON(t, ExerciseResult{Skipped: ExerciseSkippedNoQuiescedKind}, `{"skipped":"no-quiesced-kind"}`)
 }
 
 func TestAdminWire_ListAccountsResponse(t *testing.T) {

docs/DESIGN.md

@@ -65,9 +65,10 @@ The EC2 registration (`spinifex/domains/ec2/awsapi`, `NewRegistration`) parses t
 The gateway communicates with daemons via NATS request/response. Most calls go through `natsmsg.NATSRequest`, which marshals the input, attaches the account ID as a NATS header, and unmarshals the typed response:
 
 ```go
-func (s *NATSInstanceService) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
-    topic := fmt.Sprintf("ec2.RunInstances.%s", aws.StringValue(input.InstanceType))
-    return natsmsg.NATSRequest[ec2.Reservation](ctx, s.natsConn, topic, input, 5*time.Minute, accountID)
+func runIntoReservation(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, accountID, crID string) (*ec2.Reservation, error) {
+    subject := "ec2.RunInstances.cr." + crID
+    reservation, err := natsmsg.NATSRequest[ec2.Reservation](ctx, natsConn, subject, input, 5*time.Minute, accountID)
+    // ...
 }
 ```
 

docs/containers/eks/README.md

@@ -59,6 +59,9 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM
 
 - **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
 - **The cluster security group is auto-managed.** As on EKS, Spinifex creates one `eks-cluster-sg-<name>-<id>` in your VPC, tagged `kubernetes.io/cluster/<name>=owned` and `aws:eks:cluster-name`, allowing all traffic from itself. It is attached to the workers and the private-endpoint network interface, and `DescribeCluster` reports it as `clusterSecurityGroupId`. A launch template that names security groups replaces it on the workers, as on AWS. Clusters created before this change keep their `-control-plane-sg` and `-nodegroup-sg` until you recreate them. Deleting a cluster deletes the cluster security group and removes only the rules on your other security groups that reference it. `vpc_config.security_group_ids` is ignored.
+- **Node group deletes are asynchronous.** `DeleteNodegroup` returns at once with the group `DELETING`, and `DescribeNodegroup` answers `ResourceNotFoundException` once the workers and the group's instance profile are gone. Deleting a cluster or creating a group of the same name while a group is `DELETING` returns `ResourceInUseException`, as does a second delete of a `DELETING` group.
+- **If a node group delete cannot finish,** the group moves to `DELETE_FAILED` after 30 minutes, with a `NODE_TERMINATION_FAILURE` health issue naming the workers that are still running. Run `DeleteNodegroup` again to resume it.
+- **Each node group owns an EKS-managed instance profile.** Spinifex creates it, tags it `spinifex:eks-cluster`, `spinifex:eks-nodegroup` and `spinifex:eks-profile-id`, and puts the node role in it. It is removed with the group, so you can delete the node role afterwards with no extra steps. Spinifex never deletes an instance profile it did not create, or one whose tags, name or role no longer match what it recorded; it leaves the profile and logs a warning with its name.
 - **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
 - **Workers launch with IMDSv2 required and an IMDS hop limit of 2.**
   This matches AWS managed node groups and lets a pod without IRSA reach the node role through the SDK default credential chain.
@@ -377,6 +380,15 @@ Control plane readiness and kubectl authentication above lists which clusters re
 
 **`create-cluster` rejected with an authentication-mode error.** `CONFIG_MAP` is the one mode Spinifex rejects, because it has no `aws-auth` ConfigMap path. Set `authentication_mode` to `API` or `API_AND_CONFIG_MAP`, or leave it unset, and grant access with access entries.
 
+**`delete-role` fails with `DeleteConflict` after deleting a node group.** Node groups created before per-group instance profiles left an instance profile named after the role, which Spinifex no longer removes because it cannot prove it created it. Remove it yourself:
+
+```bash
+aws iam remove-role-from-instance-profile --instance-profile-name <role-name> --role-name <role-name>
+aws iam delete-instance-profile --instance-profile-name <role-name>
+```
+
+**A node group is `DELETE_FAILED`.** `aws eks describe-nodegroup` lists the workers that did not stop under `health.issues`. Once they are gone, or after you fix the cause, run `aws eks delete-nodegroup` again.
+
 **Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, use the AWS form from `DescribeAddonVersions` (see Add-on versions above), or leave `addon_version` unset so Spinifex selects the catalog default.
 
 ## Reference Architectures

docs/package-boundary/eks-cluster-lifecycle.md

@@ -34,7 +34,7 @@ Node group workers and CSI-provisioned EBS volumes are never removed by a cluste
 
 ## Gaps
 
-1. Four `eks.*` handlers still hold their request for work that grows with the resource, and the subject table names them as not conformant: `CreateCluster` (reclaiming a `FAILED` cluster of the same name tears its infra down inline), `DeleteNodegroup` (one awaited terminate per worker), `UpdateNodegroupConfig` (one awaited launch or terminate per worker added or removed) and `RestoreSnapshot` (launches a control-plane VM and fences the old one inline).
+1. Three `eks.*` handlers still hold their request for work that grows with the resource, and the subject table names them as not conformant: `CreateCluster` (reclaiming a `FAILED` cluster of the same name tears its infra down inline), `UpdateNodegroupConfig` (one awaited launch or terminate per worker added or removed) and `RestoreSnapshot` (launches a control-plane VM and fences the old one inline).
 2. AWS's answer to a repeat delete on a `DELETING` cluster has not been observed; `DELETING` from the record is the chosen behaviour.
 3. Control-plane, load balancer and VPC resources in the system account are keyed by the bare cluster name, so a teardown by name can reach another tenant's same-named cluster.
 4. The teardown deletes load balancers the in-cluster controller created in the customer account, which AWS leaves behind.

docs/vultr-integration/README.md

@@ -265,7 +265,12 @@ If this returns a list of available instance types, your installation is working
 
 Spinifex is running, but it holds nothing yet: no machine images, no networks, no instances.
 
-Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance.
+Log in to node 1, and follow [Setting Up Your Cluster](/docs/setting-up-your-cluster) there to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance:
+
+```bash
+./deploy-vultr.sh ssh
+```
+
 An instance you launch into a public subnet gets a Vultr reserved IP, which you can see in the Vultr portal under **Network → Reserved IPs**.
 
 ## 8. Test resiliency

spinifex/agents/lambda/agent/bootstrap.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os/exec"
+	"path/filepath"
 	goruntime "runtime"
 	"syscall"
 	"time"
@@ -18,8 +19,8 @@ import (
 const sandboxID = 993
 
 // StartBootstrap starts the function's bootstrap at path as the sandbox uid and
-// gid with no supplementary groups, with no_new_privs, and with exactly env.
-// The caller waits on the returned command.
+// gid with no supplementary groups, with no_new_privs, and with exactly env,
+// in the bootstrap's own directory. The caller waits on the returned command.
 func StartBootstrap(ctx context.Context, path string, env []string, stdout, stderr io.Writer) (*exec.Cmd, error) {
 	cred := &syscall.Credential{Uid: sandboxID, Gid: sandboxID, Groups: []uint32{}}
 	return startBootstrap(ctx, path, env, stdout, stderr, cred)
@@ -30,6 +31,8 @@ func startBootstrap(ctx context.Context, path string, env []string, stdout, stde
 	// A nil Env inherits the agent's environment; the bootstrap gets only env.
 	cmd.Env = append([]string{}, env...)
 	cmd.Stdout, cmd.Stderr = stdout, stderr
+	// Packaged functions open their files relative to the task root.
+	cmd.Dir = filepath.Dir(path)
 	if cred != nil {
 		cmd.SysProcAttr = &syscall.SysProcAttr{Credential: cred}
 	}

spinifex/agents/lambda/agent/bootstrap_test.go

@@ -169,3 +169,11 @@ func TestStartBootstrapMissingExecutableFails(t *testing.T) {
 	_, err := startBootstrap(context.Background(), filepath.Join(t.TempDir(), "bootstrap"), nil, nil, nil, nil)
 	assert.Error(t, err)
 }
+
+func TestStartBootstrapRunsInItsOwnDirectory(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "data.txt"), []byte("packaged"), 0o644))
+	bootstrap := filepath.Join(dir, "bootstrap")
+	require.NoError(t, os.WriteFile(bootstrap, []byte("#!/bin/sh\ncat data.txt\n"), 0o755))
+	assert.Equal(t, "packaged", string(run(t, bootstrap, []string{"PATH=/bin:/usr/bin"}, false)))
+}

spinifex/bootstrap/config/config.go

@@ -5,6 +5,7 @@ package config
 import (
 	"fmt"
 	"log/slog"
+	"maps"
 	"net/netip"
 	"os"
 	"path/filepath"
@@ -355,6 +356,11 @@ type EBSConfig struct {
 	CacheSizeMB *int `json:"CacheSizeMB" mapstructure:"cache_size_mb"`
 }
 
+// SupportedEBSProvider reports whether name is a provider a node may run.
+func SupportedEBSProvider(name string) bool {
+	return name == EBSProviderViperblockd || name == EBSProviderQEMUNBD
+}
+
 // ResolvedProvider normalizes an empty Provider to EBSProviderViperblockd,
 // the default provider.
 func (c EBSConfig) ResolvedProvider() string {
@@ -801,6 +807,27 @@ func validateVultrPool(cc *ClusterConfig, p ExternalPool) error {
 	return nil
 }
 
+// validateSingleEBSProvider refuses nodes whose explicit [nodes.<name>.ebs]
+// providers differ. A peer stanza without one is unknown, not the default: the
+// generated config omits it, and the cluster's KV lock is the authoritative check.
+func validateSingleEBSProvider(cc *ClusterConfig) error {
+	var first, want string
+	for _, name := range slices.Sorted(maps.Keys(cc.Nodes)) {
+		if !viper.IsSet("nodes." + name + ".ebs.provider") {
+			continue
+		}
+		got := cc.Nodes[name].EBS.ResolvedProvider()
+		if first == "" {
+			first, want = name, got
+			continue
+		}
+		if got != want {
+			return fmt.Errorf("config: [nodes.%s.ebs] provider resolves to %q but [nodes.%s.ebs] resolves to %q; every node in a cluster must use the same EBS provider", first, want, name, got)
+		}
+	}
+	return nil
+}
+
 func validateClusterConfig(cc *ClusterConfig) error {
 	if viper.IsSet("network.external_dhcp") {
 		return fmt.Errorf("config: [network] external_dhcp is no longer supported; remove the key (static WAN-pool allocation only)")
@@ -809,9 +836,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		if viper.IsSet("nodes." + nodeName + ".vpcd.dhcp_bind_bridge") {
 			return fmt.Errorf("config: [nodes.%s.vpcd] dhcp_bind_bridge is no longer supported; remove the key (vpcd no longer runs a DHCP client)", nodeName)
 		}
-		switch nodeCfg.EBS.Provider {
-		case "", EBSProviderViperblockd, EBSProviderQEMUNBD:
-		case EBSProviderEmbedded:
+		switch {
+		case nodeCfg.EBS.Provider == "", SupportedEBSProvider(nodeCfg.EBS.Provider):
+		case nodeCfg.EBS.Provider == EBSProviderEmbedded:
 			// Refuse rather than silently upgrade: the embedded engine is gone,
 			// and starting anyway would migrate this node's volumes to
 			// ebsmetadata one-way without the operator ever asking for it.
@@ -829,6 +856,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 			return fmt.Errorf("config: [nodes.%s.ebs] cache_size_mb=%d must not be negative; use 0 to disable the cache, or remove the key for the default of %d", nodeName, *c, DefaultCacheSizeMB)
 		}
 	}
+	if err := validateSingleEBSProvider(cc); err != nil {
+		return err
+	}
 
 	if err := validateNorthstarCluster(cc); err != nil {
 		return err

spinifex/bootstrap/config/config_test.go

@@ -1108,6 +1108,45 @@ provider = "bogus"
 	assert.Contains(t, err.Error(), `provider="bogus"`)
 }
 
+func TestLoadConfig_EBSProvider_SingleProviderPerCluster(t *testing.T) {
+	stanza := func(node, provider string) string {
+		return fmt.Sprintf("[nodes.%s.ebs]\nprovider = %q\n", node, provider)
+	}
+	cases := []struct {
+		name    string
+		ebs     []string
+		wantErr string
+	}{
+		{"all unset", []string{"", "", ""}, ""},
+		{"explicit viperblockd beside unset peers", []string{"", stanza("n2", EBSProviderViperblockd), ""}, ""},
+		{"explicit qemunbd beside unset peers", []string{stanza("n1", EBSProviderQEMUNBD), "", ""}, ""},
+		{"all explicit qemunbd", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderQEMUNBD), stanza("n3", EBSProviderQEMUNBD)}, ""},
+		{"two explicit providers differ", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderViperblockd), ""},
+			`[nodes.n1.ebs] provider resolves to "qemunbd" but [nodes.n2.ebs] resolves to "viperblockd"`},
+		{"only the third node differs", []string{stanza("n1", EBSProviderViperblockd), stanza("n2", EBSProviderViperblockd), stanza("n3", EBSProviderQEMUNBD)},
+			`[nodes.n1.ebs] provider resolves to "viperblockd" but [nodes.n3.ebs] resolves to "qemunbd"`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			body := "node = \"n1\"\n"
+			for i, ebs := range tc.ebs {
+				body += fmt.Sprintf("\n[nodes.n%d]\nregion = \"us-east-1\"\n%s", i+1, ebs)
+			}
+			resetViper(t)
+			path := filepath.Join(t.TempDir(), "spinifex.toml")
+			require.NoError(t, os.WriteFile(path, []byte(body), 0600))
+
+			_, err := LoadConfig(path)
+			if tc.wantErr == "" {
+				require.NoError(t, err)
+				return
+			}
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.wantErr)
+		})
+	}
+}
+
 func TestEBSConfig_ResolvedTunablesDefault(t *testing.T) {
 	var c EBSConfig
 	assert.Equal(t, DefaultNBDKitThreads, c.ResolvedThreads())

spinifex/domains/acm/awsapi/registration.go

@@ -3,14 +3,11 @@ package awsapi
 import (
 	"context"
 	"errors"
-	"io"
 	"log/slog"
 	"maps"
 	"net/http"
 	"slices"
-	"strings"
 
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 	"github.com/nats-io/nats.go"
@@ -66,7 +63,7 @@ var Actions = map[string]Handler{
 func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
-		ResolveAction: resolveAction,
+		ResolveAction: dispatch.ResolveTargetAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeJSON,
 		Inventory:     OperationInventory(),
@@ -81,25 +78,11 @@ func OperationInventory() dispatch.Inventory {
 	}
 }
 
-// resolveAction is the one X-Amz-Target parser telemetry and dispatch share.
-func resolveAction(r *http.Request) string {
-	return actionFromTarget(r.Header.Get("X-Amz-Target"))
-}
-
-// actionFromTarget extracts the action suffix from an X-Amz-Target header.
-// Any "<Prefix>.<Action>" or bare "<Action>" form is accepted.
-func actionFromTarget(target string) string {
-	if _, after, ok := strings.CutLast(target, "."); ok {
-		return after
-	}
-	return target
-}
-
 // dispatch resolves, scopes and authorizes the request from one read of its
 // body, then runs the action. Errors are awserrors codes the gateway renders.
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
-	action := resolveAction(r)
+	action := dispatch.ResolveTargetAction(r)
 	if action == "" {
 		return errors.New(awserrors.ErrorMissingAction)
 	}
@@ -118,7 +101,7 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 		return errors.New(awserrors.ErrorInternalError)
 	}
 
-	body, err := readBoundedBody(r)
+	body, err := dispatch.ReadBoundedBody(r)
 	if err != nil {
 		slog.ErrorContext(r.Context(), "ACM dispatch: failed to read body", "err", err)
 		return err
@@ -144,16 +127,3 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	WriteJSONResponse(w, output)
 	return nil
 }
-
-// readBoundedBody reads the body under the SigV4 payload cap, so a body read
-// ahead of the policy gate can neither bypass it nor exhaust memory.
-func readBoundedBody(r *http.Request) ([]byte, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return body, nil
-}

spinifex/domains/acm/awsapi/registration_test.go

@@ -58,18 +58,6 @@ func dispatchWith(t *testing.T, deps Deps, inv dispatch.Invocation) (*httptest.R
 	return w, NewRegistration(deps).Dispatch(w, inv)
 }
 
-func TestActionFromTarget(t *testing.T) {
-	for target, want := range map[string]string{
-		"CertificateManager.ListCertificates": "ListCertificates",
-		"a.b.ListCertificates":                "ListCertificates",
-		"ListCertificates":                    "ListCertificates",
-		"CertificateManager.":                 "",
-		"":                                    "",
-	} {
-		assert.Equal(t, want, actionFromTarget(target), "target %q", target)
-	}
-}
-
 // TestActionsScopeTableIsExhaustive is what stops the next ACM action being
 // added with a silent account-wide grant. It asserts both directions, so a
 // scope left behind by a deleted or renamed action fails too.

spinifex/domains/ec2/accountcleanup/cleanup.go

@@ -49,11 +49,13 @@ type PlacementGroupOwner interface {
 	CleanupAccount(ctx context.Context, accountID string, referenced func(groupID, groupName string) string) (*ec2placementgroup.AccountCleanup, error)
 }
 
-// LaunchTemplateOwner is the launch-template owner's account scan and residue
-// cleanup; live templates are never deleted by account cleanup.
+// LaunchTemplateOwner is the launch-template owner's account scan, residue
+// cleanup and live-template delete; a run deletes live templates only when its
+// request asks and the consumer check completed.
 type LaunchTemplateOwner interface {
 	ScanAccount(ctx context.Context, accountID string) (*ec2launchtemplate.AccountScan, error)
 	CleanupAccountResidue(ctx context.Context, accountID string, settledBefore time.Time) (*ec2launchtemplate.AccountCleanup, error)
+	DeleteLiveTemplates(ctx context.Context, accountID string, referenced func(id, name string) string) (*ec2launchtemplate.AccountTemplateDeletes, error)
 }
 
 // TagIndexOwner lists and clears the account's tag-index entries.
@@ -69,6 +71,28 @@ var (
 	_ TagIndexOwner       = (*ec2tags.TagsServiceImpl)(nil)
 )
 
+// kindPrefixes maps each cleanable kind to its resource-ID prefix in the tag index.
+var kindPrefixes = map[string]string{
+	ec2v1.PlatformKindKeyPair:        keyPairIDPrefix,
+	ec2v1.PlatformKindPlacementGroup: placementGroupIDPrefix,
+	ec2v1.PlatformKindLaunchTemplate: launchTemplateIDPrefix,
+}
+
+// selectKinds returns the kinds a request cleans, in report order. No kinds
+// means all three; a kind that is not cleanable, such as the tag index, is refused.
+func selectKinds(requested []string) ([]string, error) {
+	all := []string{ec2v1.PlatformKindKeyPair, ec2v1.PlatformKindPlacementGroup, ec2v1.PlatformKindLaunchTemplate}
+	if len(requested) == 0 {
+		return all, nil
+	}
+	for _, kind := range requested {
+		if !slices.Contains(all, kind) {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The kind is not one of key-pair, placement-group or launch-template.")
+		}
+	}
+	return slices.DeleteFunc(all, func(kind string) bool { return !slices.Contains(requested, kind) }), nil
+}
+
 // DefaultSettleWindow is how old headerless residue must be before it is
 // removed; anything younger may be a create still in flight.
 const DefaultSettleWindow = 10 * time.Minute
@@ -102,6 +126,7 @@ const (
 // CleanupPlatform deletes the account's key pairs, empty placement groups and
 // launch templates, removes settled residue and orphaned tag-index entries,
 // then re-scans every owner store to build the result. It is safe to re-run.
+// Request.Kinds limits the run to those kinds and their tag-index entries.
 func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, error) {
 	if req == nil || req.AccountID == "" {
 		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "The request must specify an account ID.")
@@ -118,27 +143,59 @@ func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, e
 	acct := req.AccountID
 	res := &Result{AccountID: acct, CorrelationRef: req.CorrelationRef}
 
-	refs := c.readReferences(ctx, acct)
-	keyRun := c.cleanKeyPairs(ctx, acct, settledBefore, refs)
-	groupRun := c.cleanPlacementGroups(ctx, acct, refs)
-	templateRun := c.cleanLaunchTemplates(ctx, acct, settledBefore)
+	kinds, err := selectKinds(req.Kinds)
+	if err != nil {
+		return nil, err
+	}
+	var prefixes []string
+	for _, kind := range kinds {
+		prefixes = append(prefixes, kindPrefixes[kind])
+	}
+
+	var keyRun *keyPairRun
+	var groupRun *placementGroupRun
+	var templateRun *launchTemplateRun
+	// Live templates go first when asked: a version that names a key pair or
+	// group blocks it, so the reference read must see what remains.
+	if req.DeleteLiveTemplates && slices.Contains(kinds, ec2v1.PlatformKindLaunchTemplate) {
+		templateRun = c.cleanLaunchTemplates(ctx, acct, settledBefore, true)
+	}
+	if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) || slices.Contains(kinds, ec2v1.PlatformKindPlacementGroup) {
+		refs := c.readReferences(ctx, acct)
+		if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) {
+			keyRun = c.cleanKeyPairs(ctx, acct, settledBefore, refs)
+		}
+		if slices.Contains(kinds, ec2v1.PlatformKindPlacementGroup) {
+			groupRun = c.cleanPlacementGroups(ctx, acct, refs)
+		}
+	}
+	if templateRun == nil && slices.Contains(kinds, ec2v1.PlatformKindLaunchTemplate) {
+		templateRun = c.cleanLaunchTemplates(ctx, acct, settledBefore, false)
+	}
 
 	// Tags are listed before the owner re-scans, so a resource created in
 	// between is seen as owned and keeps its tags.
 	tagKind := c.startKind(ec2v1.PlatformKindTagIndex)
-	tagged, tagErr := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	tagged, tagErr := c.tags.ScanAccountResourceIDs(ctx, acct, prefixes...)
 	if tagErr != nil {
 		slog.WarnContext(ctx, "Platform cleanup: tag index listing failed", "err", tagErr)
 		tagKind.incomplete(ec2v1.PlatformReasonStoreListFailed)
 	}
 
-	owners := map[string]ownedIDs{
-		keyPairIDPrefix:        c.finishKeyPairs(ctx, acct, keyRun, res),
-		placementGroupIDPrefix: c.finishPlacementGroups(ctx, acct, groupRun, res),
-		launchTemplateIDPrefix: c.finishLaunchTemplates(ctx, acct, templateRun, res),
+	owners := map[string]ownedIDs{}
+	if keyRun != nil {
+		owners[keyPairIDPrefix] = c.finishKeyPairs(ctx, acct, keyRun, res)
+		res.Kinds = append(res.Kinds, keyRun.kind.report)
 	}
-	res.Kinds = []KindReport{keyRun.kind.report, groupRun.kind.report, templateRun.kind.report}
-	res.Kinds = append(res.Kinds, c.cleanTagIndex(ctx, acct, tagKind, tagged, tagErr == nil, owners, res))
+	if groupRun != nil {
+		owners[placementGroupIDPrefix] = c.finishPlacementGroups(ctx, acct, groupRun, res)
+		res.Kinds = append(res.Kinds, groupRun.kind.report)
+	}
+	if templateRun != nil {
+		owners[launchTemplateIDPrefix] = c.finishLaunchTemplates(ctx, acct, templateRun, res)
+		res.Kinds = append(res.Kinds, templateRun.kind.report)
+	}
+	res.Kinds = append(res.Kinds, c.cleanTagIndex(ctx, acct, tagKind, tagged, prefixes, tagErr == nil, owners, res))
 
 	slog.InfoContext(ctx, "Platform cleanup completed", "accountID", acct, "correlationRef", req.CorrelationRef,
 		"items", len(res.Items), "duration_ms", otelsetup.Millis(time.Since(start)))

spinifex/domains/ec2/accountcleanup/cleanup_live_templates_test.go

@@ -0,0 +1,159 @@
+package accountcleanup
+
+//test:in-package — shares the owner fixtures and the injected clock.
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeConsumers answers what nodegroups name; a non-nil err is returned with
+// whatever partial answer it also holds.
+type fakeConsumers struct {
+	ids, names []string
+	err        error
+}
+
+func (f *fakeConsumers) AccountTemplateConsumers(context.Context, string) (*TemplateConsumers, error) {
+	out := &TemplateConsumers{TemplateIDs: map[string]bool{}, TemplateNames: map[string]bool{}}
+	markAll(out.TemplateIDs, f.ids)
+	markAll(out.TemplateNames, f.names)
+	return out, f.err
+}
+
+var _ TemplateConsumerReader = (*fakeConsumers)(nil)
+
+// liveDelete runs a cleanup that deletes live templates, with consumers as the
+// nodegroup reader (nil means none supplied).
+func (o *owners) liveDelete(t *testing.T, consumers TemplateConsumerReader) *Result {
+	t.Helper()
+	readers := o.readers()
+	readers.Consumers = consumers
+	c := New(o.keys, o.groups, o.templates, o.tags, readers)
+	c.now = func() time.Time { return time.Now().Add(time.Hour) }
+	res, err := c.CleanupPlatform(t.Context(), &Request{AccountID: testAccount, CorrelationRef: "ref-1", DeleteLiveTemplates: true})
+	require.NoError(t, err)
+	return res
+}
+
+func (o *owners) templateNamingKeyAndGroup(t *testing.T, name, key, group string) string {
+	t.Helper()
+	ltID := o.createTemplate(t, testAccount, name)
+	_, err := o.templates.CreateLaunchTemplateVersion(t.Context(), &ec2.CreateLaunchTemplateVersionInput{
+		LaunchTemplateId: aws.String(ltID),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
+			KeyName:   aws.String(key),
+			Placement: &ec2.LaunchTemplatePlacementRequest{GroupName: aws.String(group)},
+		},
+	}, testAccount)
+	require.NoError(t, err)
+	return ltID
+}
+
+func (o *owners) liveTemplateIDs(t *testing.T) []string {
+	t.Helper()
+	scan, err := o.templates.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	return scan.LiveTemplateIDs
+}
+
+// A template no nodegroup names is deleted in the same run as the key pair and
+// group its version names, which the reference check no longer sees.
+func TestCleanupPlatformDeletesAnUnnamedTemplateAndWhatItNames(t *testing.T) {
+	o := newOwners(t)
+	key := o.importKey(t, testAccount, "tkey")
+	group := o.createGroup(t, testAccount, "tgroup")
+	lt := o.templateNamingKeyAndGroup(t, "web", "tkey", "tgroup")
+
+	res := o.liveDelete(t, &fakeConsumers{})
+	assert.ElementsMatch(t, []Item{
+		item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionDeleted, ""),
+		item(key, ec2v1.PlatformKindKeyPair, DispositionDeleted, ""),
+		item(group, ec2v1.PlatformKindPlacementGroup, DispositionDeleted, ""),
+	}, res.Items)
+	assert.Empty(t, o.liveTemplateIDs(t))
+}
+
+// A template a nodegroup names, by ID or by name, is kept and blocked, and so
+// are the key pair and group its versions name.
+func TestCleanupPlatformKeepsATemplateANodegroupNames(t *testing.T) {
+	for name, consumers := range map[string]*fakeConsumers{
+		"by id":   {},
+		"by name": {names: []string{"web"}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			o := newOwners(t)
+			key := o.importKey(t, testAccount, "tkey")
+			group := o.createGroup(t, testAccount, "tgroup")
+			lt := o.templateNamingKeyAndGroup(t, "web", "tkey", "tgroup")
+			other := o.createTemplate(t, testAccount, "free")
+			if name == "by id" {
+				consumers.ids = []string{lt}
+			}
+
+			res := o.liveDelete(t, consumers)
+			assert.ElementsMatch(t, []Item{
+				item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonLaunchTemplateReference),
+				item(other, ec2v1.PlatformKindLaunchTemplate, DispositionDeleted, ""),
+				item(key, ec2v1.PlatformKindKeyPair, DispositionBlocked, ReasonLaunchTemplateReference),
+				item(group, ec2v1.PlatformKindPlacementGroup, DispositionBlocked, ReasonLaunchTemplateReference),
+			}, res.Items)
+			assert.Equal(t, []string{lt}, o.liveTemplateIDs(t))
+		})
+	}
+}
+
+// A consumer read that fails, returns a partial answer with an error, or is not
+// supplied deletes no template, key pair or group, and the kind is incomplete.
+func TestCleanupPlatformUnreadableConsumersDeleteNoTemplate(t *testing.T) {
+	cases := map[string]TemplateConsumerReader{
+		"error":        &fakeConsumers{err: errInjected},
+		"partial":      &fakeConsumers{ids: []string{"lt-0other"}, err: errInjected},
+		"not supplied": nil,
+	}
+	for name, consumers := range cases {
+		t.Run(name, func(t *testing.T) {
+			o := newOwners(t)
+			key := o.importKey(t, testAccount, "tkey")
+			lt := o.templateNamingKeyAndGroup(t, "web", "tkey", "none")
+
+			res := o.liveDelete(t, consumers)
+			assert.Contains(t, res.Items, item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonReferenceCheckFailed))
+			assert.Contains(t, res.Items, item(key, ec2v1.PlatformKindKeyPair, DispositionBlocked, ReasonLaunchTemplateReference))
+			assert.Equal(t, ec2v1.PlatformStatusScanIncomplete, statuses(res)[ec2v1.PlatformKindLaunchTemplate])
+			assert.Equal(t, []string{lt}, o.liveTemplateIDs(t))
+		})
+	}
+}
+
+// A template only a DELETING nodegroup still names is as blocked as any other:
+// the reader reports every status, so the cleanup never sees a status to trust.
+func TestCleanupPlatformKeepsATemplateADeletingNodegroupStillNames(t *testing.T) {
+	o := newOwners(t)
+	lt := o.createTemplate(t, testAccount, "draining")
+
+	res := o.liveDelete(t, &fakeConsumers{ids: []string{lt}})
+	assert.Equal(t, []Item{item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonLaunchTemplateReference)}, res.Items)
+	assert.Equal(t, []string{lt}, o.liveTemplateIDs(t))
+}
+
+// Without the request flag a supplied consumer reader changes nothing: live
+// templates are never deleted, as the exercise relies on.
+func TestCleanupPlatformLiveTemplatesSurviveWithoutTheRequestFlag(t *testing.T) {
+	o := newOwners(t)
+	lt := o.createTemplate(t, testAccount, "web")
+	readers := o.readers()
+	readers.Consumers = &fakeConsumers{}
+	c := New(o.keys, o.groups, o.templates, o.tags, readers)
+
+	res := run(t, c)
+	assert.Equal(t, []Item{item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonReferenceUnknown)}, res.Items)
+	assert.Equal(t, []string{lt}, o.liveTemplateIDs(t))
+}

spinifex/domains/ec2/accountcleanup/cleanup_test.go

@@ -775,3 +775,38 @@ func (f failingResidue) CleanupAccountResidue(ctx context.Context, acct string,
 	}
 	return &ec2launchtemplate.AccountCleanup{KeptTemplateIDs: scan.LiveTemplateIDs, OrphanBodiesFailed: scan.OrphanVersionBodies}, nil
 }
+
+// A run limited to one kind cleans that kind and its tag-index entries only:
+// the other kinds keep their resources and their tags, and are not reported.
+func TestCleanupPlatformLimitedToKindsLeavesTheRestAlone(t *testing.T) {
+	o := newOwners(t)
+	keyID := o.importKey(t, testAccount, "k1")
+	groupID := o.createGroup(t, testAccount, "g1")
+	o.tag(t, testAccount, keyID)
+
+	c := o.newCleanup()
+	c.now = func() time.Time { return time.Now().Add(time.Hour) }
+	res, err := c.CleanupPlatform(t.Context(), &Request{AccountID: testAccount, Kinds: []string{ec2v1.PlatformKindPlacementGroup}})
+	require.NoError(t, err)
+
+	assert.Equal(t, []Item{item(groupID, ec2v1.PlatformKindPlacementGroup, DispositionDeleted, "")}, res.Items)
+	assert.Equal(t, map[string]string{
+		ec2v1.PlatformKindPlacementGroup: ec2v1.PlatformStatusObservedClean,
+		ec2v1.PlatformKindTagIndex:       ec2v1.PlatformStatusObservedClean,
+	}, statuses(res))
+
+	keys, err := o.keys.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, keys.LiveKeyPairIDs, "an unselected kind is not touched")
+	tagged, err := o.tags.ScanAccountResourceIDs(t.Context(), testAccount, "key-")
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, tagged, "an unselected kind keeps its tags")
+}
+
+func TestCleanupPlatformRefusesAKindItCannotClean(t *testing.T) {
+	o := newOwners(t)
+	for _, kinds := range [][]string{{ec2v1.PlatformKindTagIndex}, {"volume"}} {
+		_, err := o.newCleanup().CleanupPlatform(t.Context(), &Request{AccountID: testAccount, Kinds: kinds})
+		require.Error(t, err, "%v", kinds)
+	}
+}

spinifex/domains/ec2/accountcleanup/kinds.go

@@ -204,13 +204,18 @@ type launchTemplateRun struct {
 	before  *ec2launchtemplate.AccountScan
 	cleaned *ec2launchtemplate.AccountCleanup
 	did     attempts
+	// deleteLive is true when this run tried to delete live templates.
+	deleteLive bool
 }
 
-func (c *Cleanup) cleanLaunchTemplates(ctx context.Context, acct string, settledBefore time.Time) *launchTemplateRun {
-	run := &launchTemplateRun{kind: c.startKind(ec2v1.PlatformKindLaunchTemplate), did: newAttempts()}
+func (c *Cleanup) cleanLaunchTemplates(ctx context.Context, acct string, settledBefore time.Time, deleteLive bool) *launchTemplateRun {
+	run := &launchTemplateRun{kind: c.startKind(ec2v1.PlatformKindLaunchTemplate), did: newAttempts(), deleteLive: deleteLive}
 	if before, err := c.templates.ScanAccount(ctx, acct); err == nil {
 		run.before = before
 	}
+	if deleteLive {
+		c.deleteLiveTemplates(ctx, acct, run)
+	}
 	cleaned, err := c.templates.CleanupAccountResidue(ctx, acct, settledBefore)
 	if err != nil {
 		slog.WarnContext(ctx, "Platform cleanup: launch-template residue cleanup failed", "err", err)
@@ -218,10 +223,50 @@ func (c *Cleanup) cleanLaunchTemplates(ctx context.Context, acct string, settled
 		return run
 	}
 	run.cleaned = cleaned
-	keepAll(run.did.kept, cleaned.KeptTemplateIDs, ReasonReferenceUnknown)
+	for _, id := range cleaned.KeptTemplateIDs {
+		if run.did.kept[id] == "" {
+			run.did.kept[id] = ReasonReferenceUnknown
+		}
+	}
 	return run
 }
 
+// deleteLiveTemplates deletes the live templates no consumer names. Without a
+// complete consumer answer nothing is deleted and every live template is kept
+// as unchecked, so an unreadable consumer never reads as "no references".
+func (c *Cleanup) deleteLiveTemplates(ctx context.Context, acct string, run *launchTemplateRun) {
+	var consumers *TemplateConsumers
+	if c.refs.Consumers != nil {
+		var err error
+		consumers, err = c.refs.Consumers.AccountTemplateConsumers(ctx, acct)
+		if err != nil {
+			slog.WarnContext(ctx, "Platform cleanup: template consumer check failed", "err", err)
+			consumers = nil
+		}
+	}
+	if consumers == nil {
+		if run.before != nil {
+			keepAll(run.did.kept, run.before.LiveTemplateIDs, ReasonReferenceCheckFailed)
+		}
+		run.kind.incomplete(ReasonReferenceCheckFailed)
+		return
+	}
+	deletes, err := c.templates.DeleteLiveTemplates(ctx, acct, func(id, name string) string {
+		if consumers.names(id, name) {
+			return ReasonLaunchTemplateReference
+		}
+		return ""
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "Platform cleanup: live template delete failed", "err", err)
+		run.kind.incomplete(ec2v1.PlatformReasonStoreListFailed)
+		return
+	}
+	markAll(run.did.deleted, deletes.DeletedTemplateIDs)
+	markAll(run.did.failed, deletes.FailedTemplateIDs)
+	maps.Copy(run.did.kept, deletes.Referenced)
+}
+
 func (c *Cleanup) finishLaunchTemplates(ctx context.Context, acct string, run *launchTemplateRun, res *Result) ownedIDs {
 	defer c.finishKind(run.kind)
 	after, err := c.templates.ScanAccount(ctx, acct)
@@ -250,8 +295,10 @@ func (c *Cleanup) finishLaunchTemplates(ctx context.Context, acct string, run *l
 	}
 	// Live templates are never deleted here, so every one still present is
 	// blocked on the unknown EKS nodegroup reference, whenever it appeared.
-	for _, id := range afterLive {
-		run.did.kept[id] = ReasonReferenceUnknown
+	if !run.deleteLive {
+		for _, id := range afterLive {
+			run.did.kept[id] = ReasonReferenceUnknown
+		}
 	}
 	for _, id := range sortedUnion(beforeLive, kept, afterLive) {
 		disposition, reason := itemDisposition(id, run.did, present[id], after != nil)
@@ -282,15 +329,15 @@ func (c *Cleanup) finishLaunchTemplates(ctx context.Context, acct string, run *l
 // cleanTagIndex clears entries whose owner re-scan resolved and holds no
 // record for the ID, then re-lists the index to prove each clear. Entries of
 // resources that still exist are theirs, not residue, and are left alone.
-func (c *Cleanup) cleanTagIndex(ctx context.Context, acct string, kind *kindRun, tagged []string, listed bool, owners map[string]ownedIDs, res *Result) KindReport {
+func (c *Cleanup) cleanTagIndex(ctx context.Context, acct string, kind *kindRun, tagged, prefixes []string, listed bool, owners map[string]ownedIDs, res *Result) KindReport {
 	if listed {
-		c.clearOrphanTags(ctx, acct, kind, tagged, owners, res)
+		c.clearOrphanTags(ctx, acct, kind, tagged, prefixes, owners, res)
 	}
 	c.finishKind(kind)
 	return kind.report
 }
 
-func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRun, tagged []string, owners map[string]ownedIDs, res *Result) {
+func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRun, tagged, prefixes []string, owners map[string]ownedIDs, res *Result) {
 	// A resource this run reports as not deleted owns its tags even when its
 	// kind's re-scan failed.
 	reported := map[string]bool{}
@@ -317,7 +364,7 @@ func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRu
 			cleared = append(cleared, id)
 		}
 	}
-	after, err := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	after, err := c.tags.ScanAccountResourceIDs(ctx, acct, prefixes...)
 	if err != nil {
 		slog.WarnContext(ctx, "Platform cleanup: tag index re-scan failed", "err", err)
 		kind.incomplete(ec2v1.PlatformReasonStoreListFailed)

spinifex/domains/ec2/accountcleanup/references.go

@@ -35,17 +35,38 @@ type TemplateReferenceReader interface {
 	AccountTemplateReferences(ctx context.Context, accountID string) (*ec2launchtemplate.TemplateReferences, error)
 }
 
+// TemplateConsumers are the IDs and names of launch templates that records
+// outside EC2, such as EKS nodegroups, still name. They select what to keep and
+// are never reported.
+type TemplateConsumers struct {
+	TemplateIDs   map[string]bool
+	TemplateNames map[string]bool
+}
+
+// TemplateConsumerReader reads which launch templates records outside EC2 name,
+// every status included, complete or an error. The consumer's owner answers it;
+// EC2 never reads those records.
+type TemplateConsumerReader interface {
+	AccountTemplateConsumers(ctx context.Context, accountID string) (*TemplateConsumers, error)
+}
+
+func (c *TemplateConsumers) names(id, name string) bool {
+	return c.TemplateIDs[id] || c.TemplateNames[name]
+}
+
 var (
 	_ SpotReferenceReader     = (*ec2spotinstance.SpotInstanceServiceImpl)(nil)
 	_ TemplateReferenceReader = (*ec2launchtemplate.LaunchTemplateServiceImpl)(nil)
 )
 
 // ReferenceReaders are the checks run before key pairs and placement groups
-// are deleted. A nil reader counts as a failed check.
+// are deleted. A nil reader counts as a failed check. Consumers is read only
+// by a request that deletes live templates, where nil is also a failed check.
 type ReferenceReaders struct {
 	Instances InstanceReferenceReader
 	Spot      SpotReferenceReader
 	Templates TemplateReferenceReader
+	Consumers TemplateConsumerReader
 }
 
 func newReferences() *References {

spinifex/domains/ec2/accountcleanup/types.go

@@ -7,6 +7,14 @@ import "time"
 type Request struct {
 	AccountID      string `json:"account_id"`
 	CorrelationRef string `json:"correlation_ref,omitempty"`
+
+	// Kinds limits the run to these platform kinds. Empty means all three.
+	Kinds []string `json:"kinds,omitempty"`
+
+	// DeleteLiveTemplates lets the run delete live launch templates that no
+	// nodegroup names, using the TemplateConsumerReader. Without it live
+	// templates are never deleted.
+	DeleteLiveTemplates bool `json:"delete_live_templates,omitempty"`
 }
 
 // Result is what the owner stores hold after one run. Kinds reuse the ec2.v1
@@ -85,7 +93,8 @@ const (
 	ReasonInstanceReference = "instance-reference"
 	// ReasonSpotRequestReference: an open or active spot request names it.
 	ReasonSpotRequestReference = "spot-request-reference"
-	// ReasonLaunchTemplateReference: a version of a live launch template names it.
+	// ReasonLaunchTemplateReference: a version of a live launch template names a
+	// key pair or group, or a nodegroup names a template.
 	ReasonLaunchTemplateReference = "launch-template-reference"
 	// ReasonReferenceUnknown: a record EC2 cannot read may name it, so it is
 	// never deleted by account cleanup.

spinifex/domains/ec2/awsapi/registration.go

@@ -1,8 +1,6 @@
 package awsapi
 
 import (
-	"net/http"
-
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 )
 
@@ -11,7 +9,7 @@ import (
 func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
-		ResolveAction: resolveAction,
+		ResolveAction: dispatch.ResolveQueryAction,
 		Dispatch:      NewHandler(deps).Dispatch,
 		Errors:        dispatch.ErrorEnvelopeXML,
 		Traits:        dispatch.ServiceTraits{XMLErrorShape: dispatch.XMLErrorShapeEC2},
@@ -24,13 +22,3 @@ func NewRegistration(deps Deps) dispatch.Registration {
 func OperationInventory() dispatch.Inventory {
 	return dispatch.Inventory{Registered: ActionNames()}
 }
-
-// resolveAction is the one Query Action parser telemetry and dispatch share.
-// It leaves the body readable for the dispatcher and answers "" on any failure.
-func resolveAction(r *http.Request) string {
-	args, err := readQueryArgs(r)
-	if err != nil {
-		return ""
-	}
-	return args["Action"]
-}

spinifex/domains/ec2/awsapi/registration_test.go

@@ -1,11 +1,6 @@
-//test:in-package — drives the unexported resolveAction with the registration.
-
 package awsapi
 
 import (
-	"net/http"
-	"net/http/httptest"
-	"strings"
 	"testing"
 
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -27,29 +22,3 @@ func TestNewRegistration_DeclaresTheEC2Service(t *testing.T) {
 	b := dispatch.NewBuilder()
 	require.NoError(t, b.Register(reg))
 }
-
-func TestResolveAction(t *testing.T) {
-	post := func(body string) *http.Request {
-		r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
-		r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-		return r
-	}
-
-	assert.Equal(t, "DescribeVpcs", resolveAction(post("Action=DescribeVpcs&Version=2016-11-15")))
-	assert.Empty(t, resolveAction(post("Version=2016-11-15")))
-
-	// The body stays readable for the dispatcher after the action is resolved.
-	r := post("Action=DescribeVpcs")
-	require.Equal(t, "DescribeVpcs", resolveAction(r))
-	assert.Equal(t, "DescribeVpcs", resolveAction(r))
-}
-
-func TestResolveAction_UnreadableBody(t *testing.T) {
-	r := httptest.NewRequest(http.MethodPost, "/", errReader{})
-	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	assert.Empty(t, resolveAction(r))
-}
-
-type errReader struct{}
-
-func (errReader) Read([]byte) (int, error) { return 0, http.ErrBodyReadAfterClose }

spinifex/domains/ec2/awsapi/request.go

@@ -1,9 +1,7 @@
 package awsapi
 
 import (
-	"bytes"
 	"errors"
-	"io"
 	"log/slog"
 	"maps"
 	"net/http"
@@ -12,7 +10,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/bluebottle/pkg/iampolicy"
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -26,25 +23,11 @@ const (
 	ec2ServicePrincipal = "ec2.amazonaws.com"
 )
 
-// readQueryArgs parses the Query body and rewinds it, so the same bytes can be
-// read again. Any read or parse failure is a malformed query string.
-func readQueryArgs(r *http.Request) (map[string]string, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, err
-	}
-	r.Body = io.NopCloser(bytes.NewReader(body))
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return query.ParseArgs(string(body))
-}
-
 // Dispatch parses, authorizes and runs one EC2 request, then writes the XML
 // response. A returned error is an awserrors code the gateway renders.
 func (h *Handler) Dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
-	queryArgs, err := readQueryArgs(r)
+	queryArgs, err := dispatch.ReadArgs(r)
 	if err != nil {
 		slog.Debug("EC2: malformed query string", "err", err)
 		return errors.New(awserrors.ErrorMalformedQueryString)

spinifex/domains/ec2/instance/service_impl.go

@@ -2689,6 +2689,15 @@ func (s *InstanceServiceImpl) capacityRefusal(ctx context.Context, instance *vm.
 // start returns it to the stopped store, a recovery returns it to the node it
 // was taken from.
 func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instance *vm.VM, undo func()) error {
+	// Before anything is stamped or claimed, so the record handed back is the
+	// one that was claimed.
+	if err := vm.CheckUEFIVarStore(instance); err != nil {
+		slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without exactly one variable store",
+			"instanceId", instance.ID, "err", err)
+		undo()
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+
 	instance.ResetNodeLocalState()
 
 	// AWS restamps LaunchTime on every start. The pending watchdog times a

spinifex/domains/ec2/instance/service_impl_test.go

@@ -2605,6 +2605,34 @@ func TestStartStoppedInstance_InstanceTypeUnknown(t *testing.T) {
 	assert.Equal(t, vm.StateStopped, store.WroteStopped[id].Status)
 }
 
+// A UEFI instance with no variable store pointer is refused before anything is
+// stamped or claimed, so the record handed back is the one that was claimed.
+func TestStartStoppedInstance_UEFIInstanceWithoutAPointerIsRefused(t *testing.T) {
+	id := "i-uefi-nopointer"
+	itype := "t3.micro"
+	launched := time.Unix(1700000000, 0).UTC()
+	stopped := &vm.VM{
+		ID: id, Status: vm.StateStopped, AccountID: "acc", InstanceType: itype, BootMode: "uefi",
+		Instance: &ec2.Instance{LaunchTime: aws.Time(launched)},
+	}
+	before := stopped.Record()
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stopped}}
+	prov := &fakeResourceCapacityProvider{instanceTypes: map[string]*ec2.InstanceTypeInfo{
+		itype: {InstanceType: aws.String(itype)},
+	}}
+	svc := &InstanceServiceImpl{stoppedStore: store, resourceMgr: prov, vmMgr: vm.NewManager()}
+
+	_, err := svc.StartStoppedInstance(context.Background(), &StartStoppedInstanceInput{InstanceID: id}, "acc")
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Empty(t, prov.allocated, "nothing is allocated for a start that cannot boot")
+	require.NotNil(t, store.WroteStopped[id], "the claimed instance is handed back")
+	assert.Equal(t, before, store.WroteStopped[id].Record(), "the record is unchanged")
+	_, inManager := svc.vmMgr.Get(id)
+	assert.False(t, inManager)
+}
+
 func TestStartStoppedInstance_AllocateFails(t *testing.T) {
 	id := "i-alloc-fail"
 	itype := "t3.micro"

spinifex/domains/ec2/instance/service_nats.go

@@ -1,80 +0,0 @@
-package instance
-
-import (
-	"context"
-	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/nats-io/nats.go"
-)
-
-// NATSInstanceService handles instance operations via NATS. Fan-out methods
-// (DescribeInstances, DescribeInstanceTypes) intentionally error — use the
-// gateway's scatter-gather helper for cluster-wide results.
-type NATSInstanceService struct {
-	natsConn *nats.Conn
-}
-
-var _ InstanceService = (*NATSInstanceService)(nil)
-
-// NewNATSInstanceService creates a new NATS-based instance service.
-func NewNATSInstanceService(conn *nats.Conn) InstanceService {
-	return &NATSInstanceService{natsConn: conn}
-}
-
-func (s *NATSInstanceService) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
-	if input == nil || input.InstanceType == nil {
-		return nil, fmt.Errorf("instance type is required")
-	}
-	topic := fmt.Sprintf("ec2.RunInstances.%s", aws.StringValue(input.InstanceType))
-	return natsmsg.NATSRequest[ec2.Reservation](ctx, s.natsConn, topic, input, 5*time.Minute, accountID)
-}
-
-// DescribeInstances is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstances(_ context.Context, _ *ec2.DescribeInstancesInput, _ string) (*ec2.DescribeInstancesOutput, error) {
-	return nil, fmt.Errorf("DescribeInstances is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-// DescribeInstanceTypes is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstanceTypes(_ context.Context, _ *ec2.DescribeInstanceTypesInput, _ string) (*DescribeInstanceTypesNodeOutput, error) {
-	return nil, fmt.Errorf("DescribeInstanceTypes is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-// DescribeInstanceStatus is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstanceStatus(_ context.Context, _ *ec2.DescribeInstanceStatusInput, _ string) (*ec2.DescribeInstanceStatusOutput, error) {
-	return nil, fmt.Errorf("DescribeInstanceStatus is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-func (s *NATSInstanceService) DescribeInstanceAttribute(ctx context.Context, input *ec2.DescribeInstanceAttributeInput, accountID string) (*ec2.DescribeInstanceAttributeOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstanceAttributeOutput](ctx, s.natsConn, "ec2.DescribeInstanceAttribute", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) DescribeStoppedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstancesOutput](ctx, s.natsConn, "ec2.DescribeStoppedInstances", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) DescribeTerminatedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstancesOutput](ctx, s.natsConn, "ec2.DescribeTerminatedInstances", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) ModifyInstanceAttribute(ctx context.Context, input *ec2.ModifyInstanceAttributeInput, accountID string) (*ec2.ModifyInstanceAttributeOutput, error) {
-	return natsmsg.NATSRequest[ec2.ModifyInstanceAttributeOutput](ctx, s.natsConn, "ec2.ModifyInstanceAttribute", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) ModifyInstanceMetadataOptions(ctx context.Context, input *ec2.ModifyInstanceMetadataOptionsInput, accountID string) (*ec2.ModifyInstanceMetadataOptionsOutput, error) {
-	return natsmsg.NATSRequest[ec2.ModifyInstanceMetadataOptionsOutput](ctx, s.natsConn, "ec2.ModifyInstanceMetadataOptions", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) StartStoppedInstance(ctx context.Context, input *StartStoppedInstanceInput, accountID string) (*StartStoppedInstanceOutput, error) {
-	return natsmsg.NATSRequest[StartStoppedInstanceOutput](ctx, s.natsConn, "ec2.StartStoppedInstance", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) TerminateStoppedInstance(ctx context.Context, input *TerminateStoppedInstanceInput, accountID string) (*TerminateStoppedInstanceOutput, error) {
-	return natsmsg.NATSRequest[TerminateStoppedInstanceOutput](ctx, s.natsConn, "ec2.TerminateStoppedInstance", input, 30*time.Second, accountID)
-}

spinifex/domains/ec2/instance/service_nats_test.go

@@ -1,364 +0,0 @@
-package instance
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/nats-io/nats-server/v2/server"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// startTestNATSServer starts an embedded NATS server for testing.
-func startTestNATSServer(t *testing.T) (*server.Server, string) {
-	t.Helper()
-	ns, nc := testutil.StartTestNATS(t)
-	_ = nc // connection managed by testutil cleanup
-	return ns, ns.ClientURL()
-}
-
-// createValidRunInstancesInput creates a valid RunInstancesInput for testing.
-func createValidRunInstancesInput() *ec2.RunInstancesInput {
-	return &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String("t3.micro"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-		KeyName:      aws.String("test-key-pair"),
-		SecurityGroupIds: []*string{
-			aws.String("sg-0123456789abcdef0"),
-		},
-		SubnetId: aws.String("subnet-6e7f829e"),
-		UserData: aws.String("#!/bin/bash\necho 'test'"),
-	}
-}
-
-// createValidReservation creates a valid ec2.Reservation response for testing.
-func createValidReservation() *ec2.Reservation {
-	return &ec2.Reservation{
-		ReservationId: aws.String("r-0123456789abcdef0"),
-		OwnerId:       aws.String("123456789012"),
-		Instances: []*ec2.Instance{
-			{
-				InstanceId:   aws.String("i-0123456789abcdef0"),
-				InstanceType: aws.String("t3.micro"),
-				ImageId:      aws.String("ami-0abcdef1234567890"),
-				State: &ec2.InstanceState{
-					Code: aws.Int64(0),
-					Name: aws.String("pending"),
-				},
-				PrivateIpAddress: aws.String("10.0.1.100"),
-				SubnetId:         aws.String("subnet-6e7f829e"),
-			},
-		},
-	}
-}
-
-// TestNATSInstanceService_RunInstances_Success tests successful RunInstances operation.
-func TestNATSInstanceService_RunInstances_Success(t *testing.T) {
-	// Skip if LOG_IGNORE is set
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	// Start test NATS server
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	// Create NATS connections
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err, "Failed to connect to NATS")
-	defer nc.Close()
-
-	// Create mock daemon subscriber that responds with valid reservation
-	// Subscribe to the per-instance-type topic (t3.micro from createValidRunInstancesInput)
-	mockReservation := createValidReservation()
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		// Validate that request is properly formatted
-		var input ec2.RunInstancesInput
-		err := json.Unmarshal(msg.Data, &input)
-		if err != nil {
-			t.Errorf("Mock daemon received invalid JSON: %v", err)
-			return
-		}
-
-		// Send back valid reservation
-		responseData, _ := json.Marshal(mockReservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err, "Failed to subscribe mock daemon")
-
-	// Create NATSInstanceService
-	service := NewNATSInstanceService(nc)
-	require.NotNil(t, service)
-
-	// Create valid input
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify success
-	require.NoError(t, err, "RunInstances should succeed")
-	require.NotNil(t, reservation, "Reservation should not be nil")
-
-	// Verify reservation contents
-	assert.Equal(t, *mockReservation.ReservationId, *reservation.ReservationId)
-	assert.Equal(t, *mockReservation.OwnerId, *reservation.OwnerId)
-	require.Len(t, reservation.Instances, 1)
-
-	instance := reservation.Instances[0]
-	assert.Equal(t, "i-0123456789abcdef0", *instance.InstanceId)
-	assert.Equal(t, "t3.micro", *instance.InstanceType)
-	assert.Equal(t, int64(0), *instance.State.Code)
-	assert.Equal(t, "pending", *instance.State.Name)
-}
-
-// TestNATSInstanceService_RunInstances_DaemonError tests error handling when daemon returns error.
-func TestNATSInstanceService_RunInstances_DaemonError(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that responds with error
-	// Subscribe to the per-instance-type topic for the requested type
-	_, err = nc.QueueSubscribe("ec2.RunInstances.invalid.type", "spinifex-workers", func(msg *nats.Msg) {
-		// Send back error response
-		errorResponse := awserrors.GenerateErrorPayload("InvalidInstanceType")
-		msg.Respond(errorResponse)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-
-	input := &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String("invalid.type"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-	}
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify error handling - error should be just the AWS error code for gateway lookup
-	require.Error(t, err, "Should return error when daemon returns error")
-	assert.Nil(t, reservation, "Reservation should be nil on error")
-	assert.Equal(t, "InvalidInstanceType", err.Error(), "Error should be the AWS error code")
-}
-
-// TestNATSInstanceService_RunInstances_NoSubscriber tests behavior when no daemon is subscribed.
-func TestNATSInstanceService_RunInstances_NoSubscriber(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// No subscriber - no daemon running
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	start := time.Now()
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-	duration := time.Since(start)
-
-	// Verify error behavior
-	// When no subscribers are available, NATS returns ErrNoResponders immediately
-	require.Error(t, err, "Should fail when no subscriber")
-	assert.Nil(t, reservation, "Reservation should be nil")
-	assert.ErrorIs(t, err, nats.ErrNoResponders, "Error should be ErrNoResponders")
-
-	// Should fail quickly (within 1 second), not wait for 30 second timeout
-	assert.Less(t, duration.Seconds(), 1.0, "Should fail immediately when no responders")
-}
-
-// TestNATSInstanceService_RunInstances_InvalidResponse tests handling of malformed responses.
-func TestNATSInstanceService_RunInstances_InvalidResponse(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that responds with invalid JSON
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		// Send back malformed JSON
-		msg.Respond([]byte(`{"invalid": json response`))
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify error handling
-	require.Error(t, err, "Should return error for invalid JSON")
-	assert.Nil(t, reservation, "Reservation should be nil")
-	assert.Contains(t, err.Error(), "failed to unmarshal", "Error should indicate unmarshal failure")
-}
-
-// TestNATSInstanceService_RunInstances_MarshalError tests handling when input marshaling fails.
-func TestNATSInstanceService_RunInstances_MarshalError(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	service := NewNATSInstanceService(nc)
-
-	// Nil input should be rejected before hitting NATS
-	reservation, err := service.RunInstances(context.Background(), nil, "123456789012")
-	require.Error(t, err, "Should handle nil input")
-	assert.Nil(t, reservation)
-	assert.Contains(t, err.Error(), "instance type is required")
-}
-
-// TestNATSInstanceService_RunInstances_MultipleInstances tests launching multiple instances.
-func TestNATSInstanceService_RunInstances_MultipleInstances(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that returns multiple instances
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		var input ec2.RunInstancesInput
-		json.Unmarshal(msg.Data, &input)
-
-		// Create reservation with count matching MinCount
-		count := int(*input.MinCount)
-		instances := make([]*ec2.Instance, count)
-		for i := range count {
-			instanceId := aws.String(aws.StringValue(aws.String(fmt.Sprintf("i-%d", i))))
-			instances[i] = &ec2.Instance{
-				InstanceId:   instanceId,
-				InstanceType: input.InstanceType,
-				ImageId:      input.ImageId,
-				State: &ec2.InstanceState{
-					Code: aws.Int64(0),
-					Name: aws.String("pending"),
-				},
-			}
-		}
-
-		reservation := &ec2.Reservation{
-			ReservationId: aws.String("r-0123456789abcdef0"),
-			Instances:     instances,
-		}
-
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-
-	// Request 3 instances
-	input := createValidRunInstancesInput()
-	input.MinCount = aws.Int64(3)
-	input.MaxCount = aws.Int64(3)
-
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify multiple instances returned
-	require.NoError(t, err)
-	require.NotNil(t, reservation)
-	assert.Len(t, reservation.Instances, 3, "Should have 3 instances")
-}
-
-// TestNATSInstanceService_RunInstances_QueueGroup tests load balancing with queue groups.
-func TestNATSInstanceService_RunInstances_QueueGroup(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create multiple mock daemon workers in same queue group
-	worker1Count := 0
-	worker2Count := 0
-
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		worker1Count++
-		reservation := createValidReservation()
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		worker2Count++
-		reservation := createValidReservation()
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Send multiple requests
-	requestCount := 10
-	for range requestCount {
-		reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-		require.NoError(t, err)
-		require.NotNil(t, reservation)
-	}
-
-	// Verify load was distributed (both workers should have handled some requests)
-	// With queue groups, NATS distributes messages across subscribers
-	t.Logf("Worker 1 handled: %d requests", worker1Count)
-	t.Logf("Worker 2 handled: %d requests", worker2Count)
-
-	assert.Positive(t, worker1Count, "Worker 1 should handle some requests")
-	assert.Positive(t, worker2Count, "Worker 2 should handle some requests")
-	assert.Equal(t, requestCount, worker1Count+worker2Count, "All requests should be handled")
-}

spinifex/domains/ec2/instance/uefistore/uefistore.go

@@ -0,0 +1,185 @@
+// Package uefistore holds the types that describe an instance's UEFI variable
+// store and the bucket of store records that carry its deletion and write-back
+// obligations past the life of the instance record.
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+const (
+	// BucketName holds one store record per instance, keyed by instance ID.
+	BucketName = "spinifex-uefi-stores"
+	// BucketVersion is the bucket's schema stamp.
+	BucketVersion = 1
+	// RecordSchemaVersion is the version every record is written at.
+	RecordSchemaVersion uint16 = 1
+
+	// StateLive is a store whose objects belong to a running or stopped instance.
+	StateLive = "live"
+	// StateDeleting is a store whose objects are to be deleted, and then the record.
+	StateDeleting = "deleting"
+
+	objectRoot = "uefi/"
+)
+
+// ErrInvalidRecord is returned for a store record that breaks its invariants,
+// whether it is about to be written or was just read.
+var ErrInvalidRecord = errors.New("uefistore: invalid store record")
+
+// FirmwareIdentity names the firmware a variable store was written under. A
+// store only boots where Arch and VarsSize match the node's firmware, because
+// pflash needs a byte-exact region; CodeSHA256 is recorded, not enforced.
+type FirmwareIdentity struct {
+	Arch       string `json:"arch"`
+	VarsSize   int64  `json:"vars_size"`
+	CodeSHA256 string `json:"code_sha256"`
+}
+
+// UEFIStoreRecord is the durable owner of one instance's variable store
+// objects. It outlives the instance record, so an obligation recorded here is
+// not lost when a terminated record expires.
+type UEFIStoreRecord struct {
+	SchemaVersion uint16         `json:"schema_version"`
+	InstanceID    string         `json:"instance_id"`
+	AccountID     string         `json:"account_id"`
+	State         string         `json:"state"`
+	WriteBack     *UEFIWriteBack `json:"write_back,omitempty"`
+}
+
+// UEFIWriteBack is a write-back that failed and is still owed. Key is the
+// object every retry puts to, so retries leave no extra objects; SeededFrom is
+// the pointer Node seeded from, which a retry must still find on the record.
+type UEFIWriteBack struct {
+	Node       string    `json:"node"`
+	Key        string    `json:"key"`
+	SeededFrom string    `json:"seeded_from"`
+	Since      time.Time `json:"since"`
+}
+
+// KeyPrefix is the object-store prefix every store object of ownerID (an
+// instance or image ID) is written under.
+func KeyPrefix(ownerID string) string {
+	return objectRoot + ownerID + "/"
+}
+
+// ownsKey reports whether key names one object directly under ownerID's prefix.
+func ownsKey(ownerID, key string) bool {
+	name, ok := strings.CutPrefix(key, KeyPrefix(ownerID))
+	return ok && name != "" && !strings.Contains(name, "/")
+}
+
+// validate checks rec against key. A write-back key outside the instance's own
+// prefix is refused because a superseded obligation deletes that key.
+func (rec *UEFIStoreRecord) validate(key string) error {
+	switch {
+	case rec.SchemaVersion > RecordSchemaVersion:
+		return fmt.Errorf("%w: %s is at schema_version %d, newer than this node understands (%d)",
+			ErrInvalidRecord, key, rec.SchemaVersion, RecordSchemaVersion)
+	case rec.SchemaVersion != RecordSchemaVersion:
+		return fmt.Errorf("%w: %s has schema_version %d", ErrInvalidRecord, key, rec.SchemaVersion)
+	case !strings.HasPrefix(rec.InstanceID, "i-"):
+		return fmt.Errorf("%w: instance_id %q is not an instance ID", ErrInvalidRecord, rec.InstanceID)
+	case rec.InstanceID != key:
+		return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
+	case rec.State != StateLive && rec.State != StateDeleting:
+		return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
+	}
+	if wb := rec.WriteBack; wb != nil {
+		switch {
+		case wb.Node == "":
+			return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
+		case wb.Since.IsZero():
+			return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
+		case !ownsKey(rec.InstanceID, wb.Key):
+			return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
+		case !ownsKey(rec.InstanceID, wb.SeededFrom):
+			return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
+		}
+	}
+	return nil
+}
+
+func bucketConfig() kvstore.Config {
+	return kvstore.Config{
+		Name:        BucketName,
+		Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
+		History:     1,
+		// Left false: a lost bucket has lost obligations, and an empty
+		// recreated one would report every store as having none.
+		RecreateIfMissing: false,
+		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
+			return migrate.DefaultRegistry.RunKV(ctx, BucketName, kv, BucketVersion)
+		},
+		Missing: "UEFI store KV bucket not initialized",
+	}
+}
+
+// Store reads and writes store records. Every change is guarded by the
+// revision the caller read, so a lost race is reported rather than retried.
+type Store struct {
+	kv *kvstore.Store[UEFIStoreRecord]
+}
+
+// NewStore returns a Store over js. The bucket is created on Open or first use.
+func NewStore(js jetstream.JetStream) *Store {
+	return &Store{kv: kvstore.New[UEFIStoreRecord](js, bucketConfig())}
+}
+
+// Open creates the bucket if it is missing and stamps its schema version.
+func (s *Store) Open(ctx context.Context) error {
+	_, err := s.kv.KV(ctx)
+	return err
+}
+
+// Get returns instanceID's record and the revision it was read at. A missing
+// record is kvstore.ErrNotFound; one that breaks its invariants is
+// ErrInvalidRecord, so a newer or corrupt record is never acted on.
+func (s *Store) Get(ctx context.Context, instanceID string) (*UEFIStoreRecord, uint64, error) {
+	rec, rev, err := s.kv.Get(ctx, instanceID)
+	if err != nil {
+		return nil, 0, err
+	}
+	if err := rec.validate(instanceID); err != nil {
+		return nil, 0, err
+	}
+	return rec, rev, nil
+}
+
+// Create writes a new record, which must be live: a store is recorded before
+// its first object is written. An existing record is kvstore.ErrExists. Every
+// write stamps rec with RecordSchemaVersion.
+func (s *Store) Create(ctx context.Context, rec *UEFIStoreRecord) (uint64, error) {
+	rec.SchemaVersion = RecordSchemaVersion
+	if rec.State != StateLive {
+		return 0, fmt.Errorf("%w: %s created in state %q, not %q", ErrInvalidRecord, rec.InstanceID, rec.State, StateLive)
+	}
+	if err := rec.validate(rec.InstanceID); err != nil {
+		return 0, err
+	}
+	return s.kv.Create(ctx, rec.InstanceID, rec)
+}
+
+// CompareAndSet replaces the record only if it is still at rev, returning
+// kvstore.ErrConflict when it is not, and the revision written otherwise.
+func (s *Store) CompareAndSet(ctx context.Context, rec *UEFIStoreRecord, rev uint64) (uint64, error) {
+	rec.SchemaVersion = RecordSchemaVersion
+	if err := rec.validate(rec.InstanceID); err != nil {
+		return 0, err
+	}
+	return s.kv.CompareAndSet(ctx, rec.InstanceID, rec, rev)
+}
+
+// CompareAndDelete removes instanceID's record only if it is still at rev,
+// returning kvstore.ErrConflict when it is not.
+func (s *Store) CompareAndDelete(ctx context.Context, instanceID string, rev uint64) error {
+	return s.kv.CompareAndDelete(ctx, instanceID, rev)
+}

spinifex/domains/ec2/instance/uefistore/uefistore_test.go

@@ -0,0 +1,285 @@
+package uefistore_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const instanceID = "i-0123456789abcdef0"
+
+func openStore(t *testing.T) (*uefistore.Store, jetstream.JetStream) {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	s := uefistore.NewStore(js)
+	require.NoError(t, s.Open(t.Context()))
+	return s, js
+}
+
+func liveRecord() *uefistore.UEFIStoreRecord {
+	return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
+}
+
+func writeBack() *uefistore.UEFIWriteBack {
+	return &uefistore.UEFIWriteBack{
+		Node:       "node-2",
+		Key:        uefistore.KeyPrefix(instanceID) + "b",
+		SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
+		Since:      time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
+	}
+}
+
+func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
+	t.Helper()
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	_, err = kv.Put(t.Context(), key, []byte(body))
+	require.NoError(t, err)
+}
+
+// The records carry obligations past the 1 h terminated-record TTL, so the
+// bucket must not expire them, and it is stamped so a newer schema is refused.
+func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
+	_, js := openStore(t)
+
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	status, err := kv.Status(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, int64(1), status.History())
+	assert.Zero(t, status.TTL(), "a TTL would expire deletion obligations before they are settled")
+
+	v, err := kvutil.ReadVersion(t.Context(), kv)
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.BucketVersion, v)
+}
+
+func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+
+	rev, err := s.Create(t.Context(), rec)
+	require.NoError(t, err)
+
+	got, gotRev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, uefistore.RecordSchemaVersion, got.SchemaVersion)
+	assert.Equal(t, rec, got)
+}
+
+// The wire names are what every release reads, so they are pinned.
+func TestRecord_WireShape(t *testing.T) {
+	s, js := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+	_, err := s.Create(t.Context(), rec)
+	require.NoError(t, err)
+
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	entry, err := kv.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{
+		"schema_version": 1,
+		"instance_id": "i-0123456789abcdef0",
+		"account_id": "111122223333",
+		"state": "live",
+		"write_back": {
+			"node": "node-2",
+			"key": "uefi/i-0123456789abcdef0/b",
+			"seeded_from": "uefi/i-0123456789abcdef0/a",
+			"since": "2026-11-03T01:02:03Z"
+		}
+	}`, string(entry.Value()))
+}
+
+func TestGet_MissingIsNotFound(t *testing.T) {
+	s, _ := openStore(t)
+	_, _, err := s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+func TestCreate_ExistingRecordIsErrExists(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+
+	dup := liveRecord()
+	dup.AccountID = "999999999999"
+	_, err = s.Create(t.Context(), dup)
+	require.ErrorIs(t, err, kvstore.ErrExists)
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, "111122223333", got.AccountID, "the second create must not overwrite the first")
+}
+
+// A store is recorded live before its first object, so a record that starts
+// out deleting has no object it could be describing.
+func TestCreate_RefusesDeleting(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.State = uefistore.StateDeleting
+
+	_, err := s.Create(t.Context(), rec)
+	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
+	s, _ := openStore(t)
+	first, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+
+	moved := liveRecord()
+	moved.WriteBack = writeBack()
+	second, err := s.CompareAndSet(t.Context(), moved, first)
+	require.NoError(t, err)
+	assert.Greater(t, second, first)
+
+	stale := liveRecord()
+	stale.State = uefistore.StateDeleting
+	_, err = s.CompareAndSet(t.Context(), stale, first)
+	require.ErrorIs(t, err, kvstore.ErrConflict)
+
+	got, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, second, rev)
+	assert.Equal(t, uefistore.StateLive, got.State)
+	assert.Equal(t, writeBack(), got.WriteBack)
+}
+
+// Two actors that read the same revision race to change it; exactly one may
+// win, so neither can overwrite an obligation the other recorded.
+func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	_, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+
+	const writers = 8
+	var (
+		wg   sync.WaitGroup
+		mu   sync.Mutex
+		wins []string
+	)
+	for i := range writers {
+		wg.Go(func() {
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
+			rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
+			if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
+				mu.Lock()
+				wins = append(wins, rec.WriteBack.Node)
+				mu.Unlock()
+			} else {
+				assert.ErrorIs(t, err, kvstore.ErrConflict)
+			}
+		})
+	}
+	wg.Wait()
+
+	require.Len(t, wins, 1)
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, wins[0], got.WriteBack.Node)
+}
+
+func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
+	s, _ := openStore(t)
+	first, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	deleting := liveRecord()
+	deleting.State = uefistore.StateDeleting
+	second, err := s.CompareAndSet(t.Context(), deleting, first)
+	require.NoError(t, err)
+
+	require.ErrorIs(t, s.CompareAndDelete(t.Context(), instanceID, first), kvstore.ErrConflict)
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.NoError(t, err, "a delete guarded on a stale revision must leave the record")
+
+	require.NoError(t, s.CompareAndDelete(t.Context(), instanceID, second))
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+// A superseded obligation deletes its recorded key, so a key outside the
+// instance's own prefix must never reach the bucket.
+func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
+	cases := map[string]func(*uefistore.UEFIStoreRecord){
+		"unknown state":            func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
+		"not an instance ID":       func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
+		"write-back with no node":  func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
+		"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
+		"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
+		},
+		"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
+		},
+		"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
+		},
+		"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
+		},
+		"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
+		},
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			s, _ := openStore(t)
+			rev, err := s.Create(t.Context(), liveRecord())
+			require.NoError(t, err)
+
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
+			mutate(rec)
+			_, err = s.CompareAndSet(t.Context(), rec, rev)
+			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+
+			got, gotRev, err := s.Get(t.Context(), instanceID)
+			require.NoError(t, err)
+			assert.Equal(t, rev, gotRev, "an invalid record must not be written")
+			assert.Nil(t, got.WriteBack)
+		})
+	}
+}
+
+// A record a newer release wrote, or one that does not describe its own key,
+// is reported rather than decoded into something a reaper would act on.
+func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
+	cases := map[string]string{
+		"newer schema":       `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"no schema":          `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"another instance":   `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
+		"unknown state":      `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
+		"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
+	}
+	for name, body := range cases {
+		t.Run(name, func(t *testing.T) {
+			s, js := openStore(t)
+			require.True(t, json.Valid([]byte(body)))
+			rawPut(t, js, instanceID, body)
+
+			_, _, err := s.Get(t.Context(), instanceID)
+			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+		})
+	}
+}

spinifex/domains/ec2/launchtemplate/account_cleanup.go

@@ -130,3 +130,51 @@ func tallyOrphan(outcome orphanOutcome, removed, failed, unsettled *int) {
 	case orphanGone, orphanOwned:
 	}
 }
+
+// AccountTemplateDeletes is what one DeleteLiveTemplates run did, by template
+// ID. It is the owner's account of its attempts, not proof of absence.
+type AccountTemplateDeletes struct {
+	DeletedTemplateIDs []string
+	FailedTemplateIDs  []string
+	// Referenced are live templates left because referenced named them, with its reason.
+	Referenced map[string]string
+}
+
+// DeleteLiveTemplates deletes the account's live templates through the same
+// owner delete as DeleteLaunchTemplate, except those referenced names. A header
+// it cannot read or decode is left for the owner scan to report. A listing
+// failure is returned and nothing is deleted.
+func (s *LaunchTemplateServiceImpl) DeleteLiveTemplates(ctx context.Context, accountID string, referenced func(id, name string) string) (*AccountTemplateDeletes, error) {
+	keys, err := kvutil.Keys(ctx, s.kv)
+	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil, fmt.Errorf("list launch-template keys: %w", err)
+	}
+	prefix := accountID + "."
+	out := &AccountTemplateDeletes{Referenced: map[string]string{}}
+	for _, key := range keys {
+		if !isHeaderKey(key, prefix) {
+			continue
+		}
+		entry, err := s.kv.Get(ctx, key)
+		if err != nil {
+			continue
+		}
+		var header LaunchTemplateHeader
+		if json.Unmarshal(entry.Value(), &header) != nil || prefix+header.LaunchTemplateId != key {
+			continue
+		}
+		if reason := referenced(header.LaunchTemplateId, header.LaunchTemplateName); reason != "" {
+			out.Referenced[header.LaunchTemplateId] = reason
+			continue
+		}
+		if err := s.deleteTemplate(ctx, accountID, &header, entry); err != nil {
+			slog.WarnContext(ctx, "Launch-template account cleanup: live template delete failed", "launchTemplateId", header.LaunchTemplateId, "err", err)
+			out.FailedTemplateIDs = append(out.FailedTemplateIDs, header.LaunchTemplateId)
+			continue
+		}
+		out.DeletedTemplateIDs = append(out.DeletedTemplateIDs, header.LaunchTemplateId)
+	}
+	slog.InfoContext(ctx, "Launch-template account live delete completed", "accountID", accountID,
+		"deleted", len(out.DeletedTemplateIDs), "failed", len(out.FailedTemplateIDs), "referenced", len(out.Referenced))
+	return out, nil
+}

spinifex/domains/ec2/vpc/external_ipam.go

@@ -122,7 +122,7 @@ func (m *ExternalIPAM) AllocateIP(ctx context.Context, region, az, purpose, allo
 	if m == nil {
 		return "", "", fmt.Errorf("external IPAM unavailable on this node: %w", errors.New(awserrors.ErrorInsufficientAddressCapacity))
 	}
-	pool := m.findPool(region, az)
+	pool := external.FindPool(m.pools, region, az)
 	if pool == nil {
 		return "", "", fmt.Errorf("no external pool available for region=%q az=%q: %w", region, az, errors.New(awserrors.ErrorInsufficientAddressCapacity))
 	}
@@ -221,30 +221,6 @@ func (m *ExternalIPAM) allocatorFor(poolName string) (external.Allocator, error)
 	return m.static, nil
 }
 
-// findPool returns the best pool for the given region/AZ using the same
-// fallback order as topology.go: AZ-scoped → region-scoped → unscoped.
-func (m *ExternalIPAM) findPool(region, az string) *external.ExternalPoolConfig {
-	for i := range m.pools {
-		p := &m.pools[i]
-		if p.AZ != "" && p.AZ == az && p.Region == region {
-			return p
-		}
-	}
-	for i := range m.pools {
-		p := &m.pools[i]
-		if p.AZ == "" && p.Region != "" && p.Region == region {
-			return p
-		}
-	}
-	for i := range m.pools {
-		p := &m.pools[i]
-		if p.Region == "" && p.AZ == "" {
-			return p
-		}
-	}
-	return nil
-}
-
 // filterStatic returns only the static pools (Source unset or "static").
 func filterStatic(pools []external.ExternalPoolConfig) []external.ExternalPoolConfig {
 	out := make([]external.ExternalPoolConfig, 0, len(pools))

spinifex/domains/ecr/auth/rotator_test.go

@@ -99,7 +99,7 @@ func TestRotateOnce_MintsAndSwaps(t *testing.T) {
 	r.rotateOnce(t.Context(), time.Now().UTC())
 
 	// A new active key took over.
-	assert.NotEqual(t, key.Kid, issuer.ActiveKid(), "issuer should sign with the rotated key")
+	assert.NotEqual(t, key.Kid, activeKid(issuer), "issuer should sign with the rotated key")
 
 	// Both old and new tokens verify (previous key retained).
 	_, err = verifier.Verify(oldTok)
@@ -142,7 +142,7 @@ func TestRotateOnce_PrunesExpiredKey(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, metas, 1, "the older key should be pruned")
 	assert.Equal(t, second.Kid, active.Kid, "newest key remains active")
-	assert.Equal(t, second.Kid, issuer.ActiveKid())
+	assert.Equal(t, second.Kid, activeKid(issuer))
 }
 
 // TestRotator_RunRotatesThenStops drives the scheduler loop on a fast interval
@@ -169,7 +169,7 @@ func TestRotator_RunRotatesThenStops(t *testing.T) {
 	stopped := make(chan struct{})
 	go func() { r.Run(ctx); close(stopped) }()
 
-	require.Eventually(t, func() bool { return issuer.ActiveKid() != key.Kid },
+	require.Eventually(t, func() bool { return activeKid(issuer) != key.Kid },
 		2*time.Second, 5*time.Millisecond, "rotator should mint a new active key")
 
 	cancel()
@@ -218,3 +218,10 @@ func TestIssuerVerifier_ConcurrentSwap(t *testing.T) {
 	}
 	<-done
 }
+
+// activeKid reads the kid the issuer currently signs with.
+func activeKid(i *Issuer) string {
+	i.mu.RLock()
+	defer i.mu.RUnlock()
+	return i.key.Kid
+}

spinifex/domains/ecr/auth/token.go

@@ -91,16 +91,6 @@ func (i *Issuer) SetActiveKey(key *SigningKey) {
 	i.mu.Unlock()
 }
 
-// ActiveKid returns the kid of the current signing key, or "" if none.
-func (i *Issuer) ActiveKid() string {
-	i.mu.RLock()
-	defer i.mu.RUnlock()
-	if i.key == nil {
-		return ""
-	}
-	return i.key.Kid
-}
-
 // Mint returns a signed ES256 JWT for p and its expiry. The kid header lets any
 // verifier resolve the public key from the shared signing-key set.
 func (i *Issuer) Mint(p Principal) (token string, expiresAt time.Time, err error) {

spinifex/domains/ecr/awsapi/gateway_image_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -25,7 +26,8 @@ import (
 // object store + memory meta) and its composed ECR JSON image-action service.
 func newImageGateway(t *testing.T) *ecrGW {
 	t.Helper()
-	meta := ecrdomain.NewMemoryMetaStore()
+	_, _, js := testutil.StartTestJetStream(t)
+	meta := ecrdomain.NewKVMetaStore(js)
 	reg := ecrregistry.NewRegistry(objectstore.NewMemoryObjectStore(), meta, ecrTestAccount)
 	gw := newECRGW(nil, awsapi.Deps{Registry: awsapi.NewRegistryActionService(reg, reg, reg, reg), LifecyclePreview: awsapi.NewLifecyclePreviewActionService(meta, reg)})
 	imageRegistries.Store(gw, reg)

spinifex/domains/ecr/awsapi/registration.go

@@ -2,14 +2,12 @@ package awsapi
 
 import (
 	"errors"
-	"io"
 	"log/slog"
 	"maps"
 	"net/http"
 	"slices"
 	"strings"
 
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	ecrauth "github.com/mulgadc/spinifex/spinifex/domains/ecr/auth"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -34,7 +32,7 @@ type Deps struct {
 func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
-		ResolveAction: resolveAction,
+		ResolveAction: dispatch.ResolveTargetAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeJSON,
 		Inventory:     OperationInventory(),
@@ -55,25 +53,11 @@ func OperationInventory() dispatch.Inventory {
 	}
 }
 
-// resolveAction is the one X-Amz-Target parser telemetry and dispatch share.
-func resolveAction(r *http.Request) string {
-	return actionFromTarget(r.Header.Get("X-Amz-Target"))
-}
-
-// actionFromTarget extracts the action suffix from an X-Amz-Target header.
-// Any "<Prefix>.<Action>" or bare "<Action>" form is accepted.
-func actionFromTarget(target string) string {
-	if _, after, ok := strings.CutLast(target, "."); ok {
-		return after
-	}
-	return target
-}
-
 // dispatch resolves, scopes and authorizes the request from one read of its
 // body, then runs the action. Errors are awserrors codes the gateway renders.
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
-	action := resolveAction(r)
+	action := dispatch.ResolveTargetAction(r)
 	if action == "" {
 		return errors.New(awserrors.ErrorMissingAction)
 	}
@@ -92,7 +76,7 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 		return errors.New(awserrors.ErrorInternalError)
 	}
 
-	body, err := readBoundedBody(r)
+	body, err := dispatch.ReadBoundedBody(r)
 	if err != nil {
 		slog.Error("ECR dispatch: failed to read body", "err", err)
 		return err
@@ -178,19 +162,6 @@ func roleIDOf(actor dispatch.Actor) string {
 	return roleID
 }
 
-// readBoundedBody reads the body under the SigV4 payload cap, so a body read
-// ahead of the policy gate can neither bypass it nor exhaust memory.
-func readBoundedBody(r *http.Request) ([]byte, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return body, nil
-}
-
 func without(values, excluded []string) []string {
 	return slices.DeleteFunc(slices.Clone(values), func(value string) bool {
 		return slices.Contains(excluded, value)

spinifex/domains/ecr/awsapi/registration_test.go

@@ -172,18 +172,6 @@ var composedCases = []struct{ action, body string }{
 	{"DeleteRepository", `{"repositoryName":"app"}`},
 }
 
-func TestActionFromTarget(t *testing.T) {
-	for target, want := range map[string]string{
-		TargetPrefix + ".CreateRepository": "CreateRepository",
-		"a.b.ListRepositories":             "ListRepositories",
-		"GetAuthorizationToken":            "GetAuthorizationToken",
-		TargetPrefix + ".":                 "",
-		"":                                 "",
-	} {
-		assert.Equal(t, want, actionFromTarget(target), "target %q", target)
-	}
-}
-
 // Every composed action must be in the Actions namespace, or dispatch rejects
 // it as InvalidAction before its resources are ever scoped.
 func TestComposedActionsAreInTheNamespace(t *testing.T) {

spinifex/domains/ecr/meta.go

@@ -5,8 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"maps"
-	"slices"
 	"sort"
 	"strings"
 	"sync"
@@ -593,295 +591,6 @@ func (s *KVMetaStore) DeleteUpload(ctx context.Context, accountID, uploadID stri
 	return nil
 }
 
-// MemoryMetaStore is an in-memory MetaStore used by tests and single-process
-// dev runs. It is safe for concurrent use.
-type MemoryMetaStore struct {
-	mu        sync.Mutex
-	repos     map[string]map[string]RepoMeta     // account -> repo -> meta
-	policies  map[string]map[string][]byte       // account -> repo -> policyText
-	lifecycle map[string]map[string][]byte       // account -> repo -> lifecyclePolicyText
-	tags      map[string]map[string]string       // account -> repo|tag -> digest
-	manifests map[string]map[string]ManifestMeta // account -> repo|digest -> meta
-	uploads   map[string]map[string]uploadRev    // account -> id -> state+rev
-}
-
-type uploadRev struct {
-	state UploadState
-	rev   uint64
-}
-
-var _ MetaStore = (*MemoryMetaStore)(nil)
-
-// NewMemoryMetaStore constructs an empty in-memory MetaStore.
-func NewMemoryMetaStore() *MemoryMetaStore {
-	return &MemoryMetaStore{
-		repos:     make(map[string]map[string]RepoMeta),
-		policies:  make(map[string]map[string][]byte),
-		lifecycle: make(map[string]map[string][]byte),
-		tags:      make(map[string]map[string]string),
-		manifests: make(map[string]map[string]ManifestMeta),
-		uploads:   make(map[string]map[string]uploadRev),
-	}
-}
-
-func (m *MemoryMetaStore) PutRepo(_ context.Context, accountID string, meta RepoMeta) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.repos[accountID] == nil {
-		m.repos[accountID] = make(map[string]RepoMeta)
-	}
-	m.repos[accountID][meta.Name] = meta
-	return nil
-}
-
-func (m *MemoryMetaStore) GetRepo(_ context.Context, accountID, repo string) (RepoMeta, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	r, ok := m.repos[accountID][repo]
-	if !ok {
-		return RepoMeta{}, ErrNotFound
-	}
-	return r, nil
-}
-
-func (m *MemoryMetaStore) ListRepos(_ context.Context, accountID string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return slices.Sorted(maps.Keys(m.repos[accountID])), nil
-}
-
-func (m *MemoryMetaStore) DeleteRepo(_ context.Context, accountID, repo string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if _, ok := m.repos[accountID][repo]; !ok {
-		return ErrNotFound
-	}
-	delete(m.repos[accountID], repo)
-	delete(m.policies[accountID], repo)
-	delete(m.lifecycle[accountID], repo)
-	for k := range m.tags[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo {
-			delete(m.tags[accountID], k)
-		}
-	}
-	for k := range m.manifests[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo {
-			delete(m.manifests[accountID], k)
-		}
-	}
-	return nil
-}
-
-func (m *MemoryMetaStore) ListManifests(_ context.Context, accountID, repo string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	var out []string
-	for k := range m.manifests[accountID] {
-		if r, d, ok := strings.Cut(k, "|"); ok && r == repo {
-			out = append(out, d)
-		}
-	}
-	sort.Strings(out)
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutRepoPolicy(_ context.Context, accountID, repo string, policyText []byte) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.policies[accountID] == nil {
-		m.policies[accountID] = make(map[string][]byte)
-	}
-	m.policies[accountID][repo] = policyText
-	return nil
-}
-
-func (m *MemoryMetaStore) GetRepoPolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.policies[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	return p, nil
-}
-
-func (m *MemoryMetaStore) DeleteRepoPolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.policies[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	delete(m.policies[accountID], repo)
-	return p, nil
-}
-
-func (m *MemoryMetaStore) PutLifecyclePolicy(_ context.Context, accountID, repo string, policyText []byte) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.lifecycle[accountID] == nil {
-		m.lifecycle[accountID] = make(map[string][]byte)
-	}
-	m.lifecycle[accountID][repo] = policyText
-	return nil
-}
-
-func (m *MemoryMetaStore) GetLifecyclePolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.lifecycle[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	return p, nil
-}
-
-func (m *MemoryMetaStore) DeleteLifecyclePolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.lifecycle[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	delete(m.lifecycle[accountID], repo)
-	return p, nil
-}
-
-func (m *MemoryMetaStore) PutTag(_ context.Context, accountID, repo, tag, digest string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.tags[accountID] == nil {
-		m.tags[accountID] = make(map[string]string)
-	}
-	m.tags[accountID][repo+"|"+tag] = digest
-	return nil
-}
-
-func (m *MemoryMetaStore) GetTag(_ context.Context, accountID, repo, tag string) (string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	d, ok := m.tags[accountID][repo+"|"+tag]
-	if !ok {
-		return "", ErrNotFound
-	}
-	return d, nil
-}
-
-func (m *MemoryMetaStore) DeleteTag(_ context.Context, accountID, repo, tag string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	key := repo + "|" + tag
-	if _, ok := m.tags[accountID][key]; !ok {
-		return ErrNotFound
-	}
-	delete(m.tags[accountID], key)
-	return nil
-}
-
-func (m *MemoryMetaStore) ListTags(_ context.Context, accountID, repo string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	var out []string
-	for k := range m.tags[accountID] {
-		if r, t, ok := strings.Cut(k, "|"); ok && r == repo {
-			out = append(out, t)
-		}
-	}
-	sort.Strings(out)
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutManifestMeta(_ context.Context, accountID, repo string, meta ManifestMeta) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.manifests[accountID] == nil {
-		m.manifests[accountID] = make(map[string]ManifestMeta)
-	}
-	m.manifests[accountID][repo+"|"+meta.Digest] = meta
-	return nil
-}
-
-func (m *MemoryMetaStore) GetManifestMeta(_ context.Context, accountID, repo, digest string) (ManifestMeta, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	meta, ok := m.manifests[accountID][repo+"|"+digest]
-	if !ok {
-		return ManifestMeta{}, ErrNotFound
-	}
-	return meta, nil
-}
-
-func (m *MemoryMetaStore) DeleteManifestMeta(_ context.Context, accountID, repo, digest string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	key := repo + "|" + digest
-	if _, ok := m.manifests[accountID][key]; !ok {
-		return ErrNotFound
-	}
-	delete(m.manifests[accountID], key)
-	return nil
-}
-
-func (m *MemoryMetaStore) ListReferrers(_ context.Context, accountID, repo, subjectDigest string) ([]ReferrerDescriptor, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	out := []ReferrerDescriptor{}
-	for k, meta := range m.manifests[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo && meta.SubjectDigest == subjectDigest {
-			out = append(out, meta.referrerDescriptor())
-		}
-	}
-	sort.Slice(out, func(i, j int) bool { return out[i].Digest < out[j].Digest })
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutUpload(_ context.Context, accountID, uploadID string, state UploadState) (uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.uploads[accountID] == nil {
-		m.uploads[accountID] = make(map[string]uploadRev)
-	}
-	rev := m.uploads[accountID][uploadID].rev + 1
-	m.uploads[accountID][uploadID] = uploadRev{state: state, rev: rev}
-	return rev, nil
-}
-
-func (m *MemoryMetaStore) GetUpload(_ context.Context, accountID, uploadID string) (UploadState, uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	u, ok := m.uploads[accountID][uploadID]
-	if !ok {
-		return UploadState{}, 0, ErrNotFound
-	}
-	return u.state, u.rev, nil
-}
-
-func (m *MemoryMetaStore) UpdateUpload(_ context.Context, accountID, uploadID string, state UploadState, rev uint64) (uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	u, ok := m.uploads[accountID][uploadID]
-	if !ok {
-		return 0, ErrNotFound
-	}
-	if u.rev != rev {
-		return 0, ErrConflict
-	}
-	u.state = state
-	u.rev = rev + 1
-	m.uploads[accountID][uploadID] = u
-	return u.rev, nil
-}
-
-func (m *MemoryMetaStore) DeleteUpload(_ context.Context, accountID, uploadID string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if _, ok := m.uploads[accountID][uploadID]; !ok {
-		return ErrNotFound
-	}
-	delete(m.uploads[accountID], uploadID)
-	return nil
-}
-
 // mapKVErr normalizes JetStream KV errors to the MetaStore error vocabulary.
 func mapKVErr(err error) error {
 	if errors.Is(err, jetstream.ErrKeyNotFound) {

spinifex/domains/ecr/meta_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -61,8 +62,8 @@ func TestKVKeyHelpers(t *testing.T) {
 	assert.Equal(t, "sha256-abc", DigestToken("sha256:abc"))
 }
 
-func TestMemoryMetaStore_RepoRoundtrip(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_RepoRoundtrip(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	_, err := m.GetRepo(context.Background(), acct, "team/app")
@@ -80,8 +81,8 @@ func TestMemoryMetaStore_RepoRoundtrip(t *testing.T) {
 	assert.Equal(t, []string{"team/app", "team/web"}, repos)
 }
 
-func TestMemoryMetaStore_TagsAndManifests(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_TagsAndManifests(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	require.NoError(t, m.PutTag(context.Background(), acct, "r", "v1", "sha256:aaa"))
@@ -112,8 +113,8 @@ func TestMemoryMetaStore_TagsAndManifests(t *testing.T) {
 	assert.ErrorIs(t, m.DeleteManifestMeta(context.Background(), acct, "r", "sha256:ccc"), ErrNotFound)
 }
 
-func TestMemoryMetaStore_UploadCAS(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_UploadCAS(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	rev, err := m.PutUpload(context.Background(), acct, "u1", UploadState{RepoName: "r"})
@@ -143,8 +144,8 @@ func TestTrimSuffixMeta(t *testing.T) {
 	assert.Equal(t, "team/app", trimSuffixMeta("repos/team/app/meta"))
 }
 
-func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_DeleteRepoAndListManifests(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	require.NoError(t, m.PutRepo(context.Background(), acct, RepoMeta{Name: "team/app", CreatedAt: time.Now()}))
@@ -156,7 +157,7 @@ func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
 
 	digs, err := m.ListManifests(context.Background(), acct, "team/app")
 	require.NoError(t, err)
-	assert.Equal(t, []string{"sha256:ccc"}, digs)
+	assert.Equal(t, []string{DigestToken("sha256:ccc")}, digs)
 
 	require.NoError(t, m.DeleteRepo(context.Background(), acct, "team/app"))
 	_, err = m.GetRepo(context.Background(), acct, "team/app")
@@ -180,50 +181,6 @@ func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
 	assert.ErrorIs(t, m.DeleteRepo(context.Background(), acct, "team/ghost"), ErrNotFound)
 }
 
-func TestMemoryMetaStore_RepoPolicy(t *testing.T) {
-	m := NewMemoryMetaStore()
-	const acct = "000000000000"
-	const policy = `{"Version":"2012-10-17"}`
-
-	_, err := m.GetRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-	_, err = m.DeleteRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-
-	require.NoError(t, m.PutRepoPolicy(context.Background(), acct, "team/app", []byte(policy)))
-	got, err := m.GetRepoPolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(got))
-
-	deleted, err := m.DeleteRepoPolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(deleted))
-	_, err = m.GetRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-}
-
-func TestMemoryMetaStore_LifecyclePolicy(t *testing.T) {
-	m := NewMemoryMetaStore()
-	const acct = "000000000000"
-	const policy = `{"rules":[]}`
-
-	_, err := m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-	_, err = m.DeleteLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-
-	require.NoError(t, m.PutLifecyclePolicy(context.Background(), acct, "team/app", []byte(policy)))
-	got, err := m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(got))
-
-	deleted, err := m.DeleteLifecyclePolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(deleted))
-	_, err = m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-}
-
 func TestManifestMeta_DecodesRecordWithoutReferrerFields(t *testing.T) {
 	var m ManifestMeta
 	require.NoError(t, json.Unmarshal([]byte(`{"digest":"sha256:ccc","mediaType":"application/vnd.oci.image.manifest.v1+json","size":10,"pushedAt":"2026-01-01T00:00:00Z","childDigests":["sha256:ddd"]}`), &m))
@@ -232,3 +189,9 @@ func TestManifestMeta_DecodesRecordWithoutReferrerFields(t *testing.T) {
 	assert.Empty(t, m.SubjectDigest)
 	assert.Empty(t, m.ArtifactType)
 }
+
+func newTestKVMetaStore(t *testing.T) *KVMetaStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	return NewKVMetaStore(js)
+}

spinifex/domains/ecr/referrers_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -30,11 +29,7 @@ func referrerMeta(digest, subject string) ManifestMeta {
 // its own referrers state.
 func referrerStores(t *testing.T) map[string]MetaStore {
 	t.Helper()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	return map[string]MetaStore{
-		"memory": NewMemoryMetaStore(),
-		"kv":     NewKVMetaStore(testutil.NewJetStream(t, nc)),
-	}
+	return map[string]MetaStore{"kv": newTestKVMetaStore(t)}
 }
 
 func TestListReferrers(t *testing.T) {

spinifex/domains/ecr/registry/lifecycle_sweep_test.go

@@ -50,7 +50,7 @@ func putPolicy(t *testing.T, reg *Registry, account, repo, policy string) {
 const untaggedExpirePolicy = `{"rules":[{"rulePriority":1,"selection":{"tagStatus":"untagged","countType":"sinceImagePushed","countUnit":"days","countNumber":7},"action":{"type":"expire"}}]}`
 
 func TestSweepRepo_SinceImagePushed(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -65,7 +65,7 @@ func TestSweepRepo_SinceImagePushed(t *testing.T) {
 }
 
 func TestSweepRepo_ImageCountMoreThan(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:a", now.AddDate(0, 0, -1), "a")
@@ -81,7 +81,7 @@ func TestSweepRepo_ImageCountMoreThan(t *testing.T) {
 }
 
 func TestSweepRepo_NoPolicy(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:x", now.AddDate(0, 0, -100))
@@ -92,7 +92,7 @@ func TestSweepRepo_NoPolicy(t *testing.T) {
 }
 
 func TestSweepRepo_Idempotent(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -105,7 +105,7 @@ func TestSweepRepo_Idempotent(t *testing.T) {
 }
 
 func TestSweepOnce_MultiAccount(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	const acct2 = "000000000001"
 	policy := `{"rules":[{"rulePriority":1,"selection":{"tagStatus":"any","countType":"imageCountMoreThan","countNumber":1},"action":{"type":"expire"}}]}`
@@ -125,7 +125,7 @@ func TestSweepOnce_MultiAccount(t *testing.T) {
 }
 
 func TestSweepOnce_AccountsError(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	s := NewLifecycleSweeper(reg, func() ([]string, error) {
 		return nil, assert.AnError
 	}, time.Hour)
@@ -133,7 +133,7 @@ func TestSweepOnce_AccountsError(t *testing.T) {
 }
 
 func TestNewLifecycleSweeper_DefaultInterval(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	s := NewLifecycleSweeper(reg, func() ([]string, error) { return nil, nil }, 0)
 	assert.Equal(t, DefaultLifecycleSweepInterval, s.interval)
 
@@ -147,7 +147,7 @@ func TestNewLifecycleSweeper_DefaultInterval(t *testing.T) {
 // TestSweeperRun drives the ticker loop: with a tiny interval and an expiring
 // image, the first tick must delete it; cancelling the context stops Run.
 func TestSweeperRun(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -184,7 +184,7 @@ func TestSweeperRun(t *testing.T) {
 // TestSweepRepo_InvalidStoredPolicy exercises the evaluate-error branch: a
 // malformed stored policy is logged and skipped, deleting nothing.
 func TestSweepRepo_InvalidStoredPolicy(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:x", now.AddDate(0, 0, -100))
@@ -198,7 +198,7 @@ func TestSweepRepo_InvalidStoredPolicy(t *testing.T) {
 // TestSweepRepo_PolicyWithoutRepo exercises the ListImages-not-found branch: a
 // policy stored for a repo with no manifests yields nothing to expire.
 func TestSweepRepo_PolicyWithoutRepo(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	putPolicy(t, reg, testAccount, "ghost", untaggedExpirePolicy)
 

spinifex/domains/ecr/registry/registry_ensurebucket_test.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"testing"
 
-	"github.com/mulgadc/spinifex/spinifex/domains/ecr"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -32,7 +31,7 @@ func (c *countingStore) EnsureBucket(ctx context.Context, bucket string) error {
 
 func TestRegistry_EnsureBucketFailureReturns500(t *testing.T) {
 	store := &countingStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), ensureErr: errors.New("predastore down")}
-	reg := NewRegistry(store, ecr.NewMemoryMetaStore(), testAccount)
+	reg := NewRegistry(store, newTestMetaStore(t), testAccount)
 
 	w := do(reg, http.MethodGet, "/v2/myrepo/tags/list", nil, nil)
 	assert.Equal(t, http.StatusInternalServerError, w.Code)
@@ -40,7 +39,7 @@ func TestRegistry_EnsureBucketFailureReturns500(t *testing.T) {
 
 func TestRegistry_EnsureBucketCachedAfterSuccess(t *testing.T) {
 	store := &countingStore{MemoryObjectStore: objectstore.NewMemoryObjectStore()}
-	reg := NewRegistry(store, ecr.NewMemoryMetaStore(), testAccount)
+	reg := NewRegistry(store, newTestMetaStore(t), testAccount)
 
 	do(reg, http.MethodGet, "/v2/myrepo/tags/list", nil, nil)
 	do(reg, http.MethodGet, "/v2/other/tags/list", nil, nil)

spinifex/domains/ecr/registry/registry_image_test.go

@@ -44,7 +44,7 @@ func storeLayersManifest(t *testing.T, reg *Registry, repo, tag string, layers .
 }
 
 func TestDeleteImage_ReclaimsBlobsKeepsShared(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	shared := pushBlob(t, reg, repo, []byte("shared-layer-bytes"))
 	exclusive := pushBlob(t, reg, repo, []byte("exclusive-layer-bytes"))
@@ -85,7 +85,7 @@ func seedImage(t *testing.T, reg *Registry, repo, tag string) ([]byte, string) {
 }
 
 func TestStoreAndGetManifest_RoundTrip(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	manifest, digest := seedImage(t, reg, "team/app", "v1")
 
 	// Fetch by tag.
@@ -114,7 +114,7 @@ func TestStoreAndGetManifest_RoundTrip(t *testing.T) {
 }
 
 func TestListImages_TaggedAndUntagged(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	seedImage(t, reg, repo, "v1")
 	seedImage(t, reg, repo, "v2")
@@ -176,7 +176,7 @@ func TestListImages_KVDigestForm(t *testing.T) {
 }
 
 func TestDeleteImage_ByTagAndByDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	_, digest := seedImage(t, reg, repo, "v1")
 	seedImage(t, reg, repo, "v2") // shares nothing; distinct digest
@@ -205,7 +205,7 @@ func TestDeleteImage_ByTagAndByDigest(t *testing.T) {
 }
 
 func TestDeleteImage_ByDigestClearsTags(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	_, digest := seedImage(t, reg, repo, "v1")
 	// Add a second tag pointing at the same digest.
@@ -221,7 +221,7 @@ func TestDeleteImage_ByDigestClearsTags(t *testing.T) {
 }
 
 func TestStoreManifest_ImmutableTag(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	manA := fmt.Appendf(nil, `{"schemaVersion":2,"mediaType":"%s","layers":[{"digest":"%s"}]}`,
 		mediaTypeDockerManifest, pushBlob(t, reg, repo, []byte("layer-a")))
@@ -254,7 +254,7 @@ func TestStoreManifest_ImmutableTag(t *testing.T) {
 }
 
 func TestStoreManifest_BadDigestReference(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfgDg := pushBlob(t, reg, repo, []byte("cfg"))
 	manifest := fmt.Appendf(nil,

spinifex/domains/ecr/registry/registry_referrer_test.go

@@ -38,7 +38,7 @@ func storedMeta(t *testing.T, reg *Registry, body string) (subject, artifactType
 }
 
 func TestPutManifest_SubjectRecordedAndAdvertised(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 
 	tests := []struct {
@@ -65,7 +65,7 @@ func TestPutManifest_SubjectRecordedAndAdvertised(t *testing.T) {
 }
 
 func TestPutManifest_IndexSubject(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"manifests":[],"subject":{"mediaType":%q,"digest":%q,"size":10}}`,
@@ -81,7 +81,7 @@ func TestPutManifest_IndexSubject(t *testing.T) {
 }
 
 func TestPutManifest_NoSubjectLeavesFieldsEmpty(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"artifactType":%q,"config":{"mediaType":%q,"digest":%q,"size":3},"layers":[]}`,
 		mediaTypeOCIManifest, sigArtifactType, sigConfigType, cfg)
@@ -97,7 +97,7 @@ func TestPutManifest_NoSubjectLeavesFieldsEmpty(t *testing.T) {
 }
 
 func TestPutManifest_MalformedSubjectDigestRejected(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := imageWithSubject(cfg, "sha256:nothex", "")
 
@@ -111,7 +111,7 @@ func TestPutManifest_MalformedSubjectDigestRejected(t *testing.T) {
 }
 
 func TestPutManifest_IndexWithContentTypeParameterGetsNoConfigArtifactType(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"manifests":[],"subject":{"mediaType":%q,"digest":%q,"size":10}}`,
@@ -128,7 +128,7 @@ func TestPutManifest_IndexWithContentTypeParameterGetsNoConfigArtifactType(t *te
 }
 
 func TestDeleteReferrerManifest_RemovesItFromTheSubjectListing(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := imageWithSubject(cfg, absentSubjectDgt, sigArtifactType)
 	dg := digestOf([]byte(body))
@@ -183,7 +183,7 @@ func getReferrersBody(t *testing.T, reg *Registry, path string) (*httptest.Respo
 }
 
 func TestGetReferrers_ListsReferrersAsAnOCIIndex(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dgA := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	dgB := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, "application/vnd.example.sbom.v1", "cfgB")
 
@@ -202,7 +202,7 @@ func TestGetReferrers_ListsReferrersAsAnOCIIndex(t *testing.T) {
 }
 
 func TestGetReferrers_NoReferrersIsAnEmptyIndexNotAnError(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	w := do(reg, http.MethodGet, "/v2/"+referrerRepo+"/referrers/"+absentSubjectDgt, nil, nil)
@@ -212,7 +212,7 @@ func TestGetReferrers_NoReferrersIsAnEmptyIndexNotAnError(t *testing.T) {
 }
 
 func TestGetReferrers_ArtifactTypeFilter(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dgSig := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	pushReferrer(t, reg, referrerRepo, absentSubjectDgt, "application/vnd.example.sbom.v1", "cfgB")
 
@@ -230,7 +230,7 @@ func TestGetReferrers_ArtifactTypeFilter(t *testing.T) {
 }
 
 func TestGetReferrers_MalformedDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	w := do(reg, http.MethodGet, "/v2/"+referrerRepo+"/referrers/not-a-digest", nil, nil)
@@ -239,14 +239,14 @@ func TestGetReferrers_MalformedDigest(t *testing.T) {
 }
 
 func TestGetReferrers_UnknownRepository(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/ghost/referrers/"+absentSubjectDgt, nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "NAME_UNKNOWN")
 }
 
 func TestGetReferrers_DoesNotCrossRepositories(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	seedRepo(t, reg, "team/other")
 
@@ -256,7 +256,7 @@ func TestGetReferrers_DoesNotCrossRepositories(t *testing.T) {
 }
 
 func TestReferrersRoute_OnlyGET(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 	for _, method := range []string{http.MethodPut, http.MethodDelete, http.MethodPost, http.MethodPatch} {
 		w := do(reg, method, "/v2/"+referrerRepo+"/referrers/"+absentSubjectDgt, []byte("{}"), nil)
@@ -272,7 +272,7 @@ func (failingReferrerStore) ListReferrers(context.Context, string, string, strin
 }
 
 func TestGetReferrers_MetadataFailureIsA5xxNotA404(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 	reg.Meta = failingReferrerStore{reg.Meta}
 
@@ -310,7 +310,7 @@ func referrerDigests(t *testing.T, reg *Registry, repo, subject string) []string
 }
 
 func TestDeleteManifest_SubjectDeleteRemovesItsReferrers(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	sbom := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "sbom")
@@ -326,7 +326,7 @@ func TestDeleteManifest_SubjectDeleteRemovesItsReferrers(t *testing.T) {
 }
 
 func TestDeleteManifest_NestedReferrersAreRemoved(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	sigOfSig := pushReferrer(t, reg, referrerRepo, sig, sigArtifactType, "sig-of-sig")
@@ -341,7 +341,7 @@ func TestDeleteManifest_NestedReferrersAreRemoved(t *testing.T) {
 }
 
 func TestDeleteManifest_ReferrerDeleteLeavesTheSubject(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	other := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "sbom")
@@ -355,7 +355,7 @@ func TestDeleteManifest_ReferrerDeleteLeavesTheSubject(t *testing.T) {
 }
 
 func TestDeleteManifest_TagDeleteDoesNotCascade(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	require.NoError(t, reg.Meta.PutTag(context.Background(), testAccount, referrerRepo, "v1", subject))
@@ -368,7 +368,7 @@ func TestDeleteManifest_TagDeleteDoesNotCascade(t *testing.T) {
 }
 
 func TestDeleteManifest_CascadeStaysInsideTheRepository(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	elsewhere := pushReferrer(t, reg, "team/other", subject, sigArtifactType, "sig-other")
@@ -394,7 +394,7 @@ func (s failingDeleteStore) DeleteManifestMeta(ctx context.Context, account, rep
 }
 
 func TestDeleteManifest_ReferrerFailureDoesNotFailTheSubjectDelete(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	stuck := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "stuck")
 	fine := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "fine")

spinifex/domains/ecr/registry/registry_test.go

@@ -2,6 +2,7 @@ package ecrregistry
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/internal/testkit"
 
 	"crypto/sha256"
 	"encoding/hex"
@@ -21,8 +22,15 @@ import (
 
 const testAccount = "000000000000"
 
-func newTestRegistry() *Registry {
-	return NewRegistry(objectstore.NewMemoryObjectStore(), ecr.NewMemoryMetaStore(), testAccount)
+func newTestRegistry(t *testing.T) *Registry {
+	t.Helper()
+	return NewRegistry(objectstore.NewMemoryObjectStore(), newTestMetaStore(t), testAccount)
+}
+
+func newTestMetaStore(t *testing.T) *ecr.KVMetaStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	return ecr.NewKVMetaStore(js)
 }
 
 func digestOf(b []byte) string {
@@ -69,7 +77,7 @@ func pushBlob(t *testing.T, reg *Registry, repo string, data []byte) string {
 }
 
 func TestBlobUpload_Monolithic_And_Head_Get(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("hello layer bytes")
 	dg := pushBlob(t, reg, "team/app", data)
 
@@ -84,20 +92,20 @@ func TestBlobUpload_Monolithic_And_Head_Get(t *testing.T) {
 }
 
 func TestBlobHead_NotFound(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodHead, "/v2/team/app/blobs/"+digestOf([]byte("absent")), nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }
 
 func TestBlobGet_MalformedDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/blobs/sha256:zzz", nil, nil)
 	assert.Equal(t, http.StatusBadRequest, w.Code)
 	assertCode(t, w, "DIGEST_INVALID")
 }
 
 func TestBlobUpload_Chunked(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	c1 := []byte("first-chunk-")
 	c2 := []byte("second-chunk")
@@ -124,7 +132,7 @@ func TestBlobUpload_Chunked(t *testing.T) {
 }
 
 func TestBlobUpload_DigestMismatch(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	w := do(reg, http.MethodPost, "/v2/r/repo/blobs/uploads/", nil, nil)
 	loc := w.Header().Get("Location")
@@ -134,7 +142,7 @@ func TestBlobUpload_DigestMismatch(t *testing.T) {
 }
 
 func TestBlobUpload_Cancel(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	w := do(reg, http.MethodPost, "/v2/r/repo/blobs/uploads/", nil, nil)
 	loc := w.Header().Get("Location")
@@ -161,7 +169,7 @@ func putManifestReferencing(t *testing.T, reg *Registry, repo, digest string) {
 }
 
 func TestBlobMount_HitAndMiss(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("shared-layer")
 	dg := pushBlob(t, reg, "team/source", data)
 	putManifestReferencing(t, reg, "team/source", dg)
@@ -185,7 +193,7 @@ func TestBlobMount_HitAndMiss(t *testing.T) {
 // as a plain upload start, matching the authorization middleware's
 // classification of the same request.
 func TestBlobMount_MissingFromFallsThroughToUpload(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("shared-layer")
 	dg := pushBlob(t, reg, "team/source", data)
 	putManifestReferencing(t, reg, "team/source", dg)
@@ -200,7 +208,7 @@ func TestBlobMount_MissingFromFallsThroughToUpload(t *testing.T) {
 // named source repository is rejected as a mount — proving digest existence
 // alone is not enough to mount cross-repo.
 func TestBlobMount_UnreferencedDigestFallsThrough(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	unreferenced := pushBlob(t, reg, "team/other", []byte("not-in-source"))
 	seedRepo(t, reg, "team/source")
 	seedRepo(t, reg, "team/dest")
@@ -214,7 +222,7 @@ func TestBlobMount_UnreferencedDigestFallsThrough(t *testing.T) {
 // repository that does not exist never mounts, and never distinguishes
 // itself from an unreferenced-digest miss (both fall through identically).
 func TestBlobMount_NonexistentSourceFallsThrough(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dg := pushBlob(t, reg, "team/elsewhere", []byte("some-bytes"))
 	seedRepo(t, reg, "team/dest")
 
@@ -224,7 +232,7 @@ func TestBlobMount_NonexistentSourceFallsThrough(t *testing.T) {
 }
 
 func TestManifest_PutGetHead_ImageManifest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 
 	cfg := []byte(`{"config":true}`)
@@ -255,7 +263,7 @@ func TestManifest_PutGetHead_ImageManifest(t *testing.T) {
 }
 
 func TestManifest_Put_MissingBlob(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	seedRepo(t, reg, repo)
 	manifest := fmt.Appendf(nil,
@@ -268,7 +276,7 @@ func TestManifest_Put_MissingBlob(t *testing.T) {
 }
 
 func TestManifest_Get_AcceptMismatch(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -285,7 +293,7 @@ func TestManifest_Get_AcceptMismatch(t *testing.T) {
 // acceptable manifest media type as its own Accept header line rather than one
 // comma-joined value. Negotiation must consider every line, not just the first.
 func TestManifest_Get_MultipleAcceptHeaders(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -305,7 +313,7 @@ func TestManifest_Get_MultipleAcceptHeaders(t *testing.T) {
 }
 
 func TestManifest_Get_Unknown(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/manifests/missing", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "MANIFEST_UNKNOWN")
@@ -314,7 +322,7 @@ func TestManifest_Get_Unknown(t *testing.T) {
 // TestStartUpload_RepoNotCreated asserts a blob upload to a repository that was
 // never created is rejected up front — ECR does not auto-create on push.
 func TestStartUpload_RepoNotCreated(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodPost, "/v2/team/ghost/blobs/uploads/", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "NAME_UNKNOWN")
@@ -323,7 +331,7 @@ func TestStartUpload_RepoNotCreated(t *testing.T) {
 // TestPutManifest_RepoNotCreated asserts a manifest PUT to an uncreated
 // repository is rejected with NAME_UNKNOWN.
 func TestPutManifest_RepoNotCreated(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	manifest := fmt.Appendf(nil, `{"mediaType":"%s","layers":[]}`, mediaTypeDockerManifest)
 	w := do(reg, http.MethodPut, "/v2/team/ghost/manifests/v1", manifest,
 		map[string]string{"Content-Type": mediaTypeDockerManifest})
@@ -332,7 +340,7 @@ func TestPutManifest_RepoNotCreated(t *testing.T) {
 }
 
 func TestManifest_ImageIndex_ChildValidation(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 
 	// Push a child image manifest first.
@@ -372,7 +380,7 @@ func TestManifest_ImageIndex_ChildValidation(t *testing.T) {
 }
 
 func TestManifest_Delete_Tag(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -390,7 +398,7 @@ func TestManifest_Delete_Tag(t *testing.T) {
 // TestManifest_Delete_ByDigest proves delete-by-digest is no longer a no-op: it
 // removes the manifest record + object, and a subsequent GET is a 404.
 func TestManifest_Delete_ByDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("cfg-bytes"))
 	manifest := fmt.Appendf(nil,
@@ -412,7 +420,7 @@ func TestManifest_Delete_ByDigest(t *testing.T) {
 }
 
 func TestTagsList(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -434,13 +442,13 @@ func TestTagsList(t *testing.T) {
 }
 
 func TestTagsList_UnknownRepo(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/ghost/tags/list", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }
 
 func TestCatalog(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	pushBlob(t, reg, "team/a", []byte("a"))
 	pushBlob(t, reg, "team/b", []byte("b"))
 
@@ -454,21 +462,21 @@ func TestCatalog(t *testing.T) {
 }
 
 func TestDispatch_InvalidRepoName(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/Team/App/tags/list", nil, nil)
 	assert.Equal(t, http.StatusBadRequest, w.Code)
 	assertCode(t, w, "NAME_INVALID")
 }
 
 func TestDispatch_RepeatedSeparatorRepoName(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dg := pushBlob(t, reg, "team/a__b--c", []byte("layer"))
 	w := do(reg, http.MethodHead, "/v2/team/a__b--c/blobs/"+dg, nil, nil)
 	assert.Equal(t, http.StatusOK, w.Code)
 }
 
 func TestDispatch_UnknownPath(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/frobnicate", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }

spinifex/domains/ecr/registry/registry_upload_test.go

@@ -35,7 +35,7 @@ func (s *blobPutFailStore) PutObject(ctx context.Context, input *s3.PutObjectInp
 func TestFinishUpload_StoreFailure_CleansUp(t *testing.T) {
 	mem := objectstore.NewMemoryObjectStore()
 	store := &blobPutFailStore{MemoryObjectStore: mem}
-	meta := ecr.NewMemoryMetaStore()
+	meta := newTestMetaStore(t)
 	reg := NewRegistry(store, meta, testAccount)
 	seedRepo(t, reg, "team/app")
 
@@ -72,7 +72,7 @@ func awsString(s string) *string { return &s }
 // updates issued from the same observed revision cannot both commit. One wins,
 // the other gets ErrConflict — so a racing PATCH is rejected, never merged.
 func TestUploadCAS_SameRevisionSingleWinner(t *testing.T) {
-	meta := ecr.NewMemoryMetaStore()
+	meta := newTestMetaStore(t)
 	rev, err := meta.PutUpload(context.Background(), testAccount, "u-race", ecr.UploadState{RepoName: "r"})
 	require.NoError(t, err)
 
@@ -116,7 +116,7 @@ func TestUploadCAS_SameRevisionSingleWinner(t *testing.T) {
 // hash state and the recorded byte object never desynchronise.
 func TestPatchUpload_ConcurrentNoCorruption(t *testing.T) {
 	for iter := range 25 {
-		reg := newTestRegistry()
+		reg := newTestRegistry(t)
 		repo := "team/race"
 		seedRepo(t, reg, repo)
 

spinifex/domains/ecr/service_impl.go

@@ -16,8 +16,7 @@ type MetaServiceImpl struct {
 
 var _ MetaService = (*MetaServiceImpl)(nil)
 
-// NewMetaServiceImpl builds a MetaService over an explicit MetaStore (used by
-// daemon-side tests with a MemoryMetaStore).
+// NewMetaServiceImpl builds a MetaService over an explicit MetaStore.
 func NewMetaServiceImpl(store MetaStore) *MetaServiceImpl {
 	return &MetaServiceImpl{store: store}
 }

spinifex/domains/ecr/service_test.go

@@ -21,7 +21,7 @@ const svcTestAccount = "000000000000"
 // ErrNotFound/ErrConflict into the response flags rather than transport errors,
 // so they survive the AWS-error-code-only NATS reply.
 func TestMetaServiceImpl_FlagMapping(t *testing.T) {
-	svc := NewMetaServiceImpl(NewMemoryMetaStore())
+	svc := NewMetaServiceImpl(newTestKVMetaStore(t))
 
 	// Absent repo -> Found:false, nil error.
 	dr, err := svc.RepoDescribe(context.Background(), &RepoDescribeRequest{Repo: "ghost"}, svcTestAccount)

spinifex/domains/ecr/storage.go

@@ -28,42 +28,22 @@ func AccountBucket(accountID string) string {
 // shapes every ECR storage path must use. Blobs are pooled per account for
 // dedup; manifests and tags are indexed per repository.
 const (
-	// repoMetaKeyFmt is the repo config object (scan-on-push, tag mutability).
-	repoMetaKeyFmt = "repos/%s/_meta.json"
-	// tagKeyFmt maps a tag to a digest pointer (body is "sha256:...").
-	tagKeyFmt = "repos/%s/manifests/by-tag/%s"
 	// manifestKeyFmt holds manifest JSON keyed by its own digest.
 	manifestKeyFmt = "repos/%s/manifests/by-digest/%s"
-	// indexKeyFmt is the rolling tag->digest index used by ListImages.
-	indexKeyFmt = "repos/%s/manifests/index.json"
 	// blobKeyFmt is the account-wide blob pool, two-char hex shard.
 	blobKeyFmt = "blobs/%s/%s"
-	// uploadKeyFmt holds in-progress chunked upload bytes.
-	uploadKeyFmt = "uploads/%s"
 	// uploadChunkKeyFmt holds a single revision's assembled bytes under a unique
 	// per-write token, so a CAS-winning PATCH always points at exactly the bytes
 	// it hashed (a losing concurrent write lands on a different, orphaned token).
 	uploadChunkKeyFmt = "uploads/%s/%s"
 )
 
-// RepoMetaKey returns the key for a repository's config object.
-func RepoMetaKey(repo string) string { return fmt.Sprintf(repoMetaKeyFmt, repo) }
-
-// TagKey returns the key for a tag's digest pointer.
-func TagKey(repo, tag string) string { return fmt.Sprintf(tagKeyFmt, repo, tag) }
-
 // ManifestKey returns the key for a manifest stored by its digest. The digest
 // is sanitized via DigestToken so the object key carries no ':' separator.
 func ManifestKey(repo, digest string) string {
 	return fmt.Sprintf(manifestKeyFmt, repo, DigestToken(digest))
 }
 
-// IndexKey returns the key for a repository's tag->digest index.
-func IndexKey(repo string) string { return fmt.Sprintf(indexKeyFmt, repo) }
-
-// UploadKey returns the key for an in-progress upload's bytes.
-func UploadKey(uploadID string) string { return fmt.Sprintf(uploadKeyFmt, uploadID) }
-
 // UploadChunkKey returns the key for one revision's assembled upload bytes,
 // addressed by a unique per-write token.
 func UploadChunkKey(uploadID, token string) string {

spinifex/domains/ecr/storage_test.go

@@ -17,11 +17,7 @@ func TestKeyHelpers(t *testing.T) {
 		got  string
 		want string
 	}{
-		{"repo meta", RepoMetaKey("team/app"), "repos/team/app/_meta.json"},
-		{"tag", TagKey("team/app", "v1"), "repos/team/app/manifests/by-tag/v1"},
 		{"manifest", ManifestKey("team/app", "sha256:abcd"), "repos/team/app/manifests/by-digest/sha256-abcd"},
-		{"index", IndexKey("team/app"), "repos/team/app/manifests/index.json"},
-		{"upload", UploadKey("u-1"), "uploads/u-1"},
 	}
 	for _, c := range cases {
 		if c.got != c.want {

spinifex/domains/ecs/awsapi/registration.go

@@ -2,14 +2,11 @@ package awsapi
 
 import (
 	"errors"
-	"io"
 	"log/slog"
 	"maps"
 	"net/http"
 	"slices"
-	"strings"
 
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/passrole"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -35,7 +32,7 @@ type Deps struct {
 func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
-		ResolveAction: resolveAction,
+		ResolveAction: dispatch.ResolveTargetAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeJSON,
 		Inventory:     OperationInventory(),
@@ -50,35 +47,11 @@ func OperationInventory() dispatch.Inventory {
 	}
 }
 
-func resolveAction(r *http.Request) string {
-	return actionFromTarget(r.Header.Get("X-Amz-Target"))
-}
-
-// actionFromTarget extracts the action suffix from an X-Amz-Target header.
-// Any "<Prefix>.<Action>" or bare "<Action>" form is accepted.
-func actionFromTarget(target string) string {
-	if _, after, ok := strings.CutLast(target, "."); ok {
-		return after
-	}
-	return target
-}
-
-func readBoundedBody(r *http.Request) ([]byte, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return body, nil
-}
-
 // dispatch resolves, scopes and authorizes the request, then runs the action.
 // Errors are awserrors codes the gateway renders.
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
-	action := resolveAction(r)
+	action := dispatch.ResolveTargetAction(r)
 	if action == "" {
 		return errors.New(awserrors.ErrorMissingAction)
 	}
@@ -99,7 +72,7 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 		return errors.New(awserrors.ErrorInternalError)
 	}
 
-	body, err := readBoundedBody(r)
+	body, err := dispatch.ReadBoundedBody(r)
 	if err != nil {
 		slog.ErrorContext(r.Context(), "ECS dispatch: failed to read body", "err", err)
 		return err

spinifex/domains/ecs/awsapi/registration_target_test.go

@@ -1,13 +0,0 @@
-package awsapi
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestActionFromTarget(t *testing.T) {
-	assert.Equal(t, "RunTask", actionFromTarget("AmazonEC2ContainerServiceV20141113.RunTask"))
-	assert.Equal(t, "CreateCluster", actionFromTarget("CreateCluster"))
-	assert.Empty(t, actionFromTarget(""))
-}

spinifex/domains/ecs/capacity_providers.go

@@ -3,6 +3,7 @@ package ecs
 import (
 	"context"
 	"errors"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -23,7 +24,7 @@ func (s *Service) PutClusterCapacityProviders(ctx context.Context, input *ecs.Pu
 		return nil, err
 	}
 	var rec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -33,7 +34,7 @@ func (s *Service) PutClusterCapacityProviders(ctx context.Context, input *ecs.Pu
 
 	rec.CapacityProviders = awsStringSlice(input.CapacityProviders)
 	rec.DefaultCapacityProviderStrategy = strategyFromAWS(input.DefaultCapacityProviderStrategy)
-	if err := putJSON(ctx, kv, ClusterMetaKey(cluster), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, ClusterMetaKey(cluster), &rec); err != nil {
 		return nil, err
 	}
 	return &ecs.PutClusterCapacityProvidersOutput{Cluster: rec.toAWS()}, nil
@@ -53,7 +54,7 @@ func (s *Service) CreateCapacityProvider(ctx context.Context, input *ecs.CreateC
 	}
 
 	var existing CapacityProviderRecord
-	found, err := getJSON(ctx, kv, CapacityProviderKey(name), &existing)
+	found, err := taskdefinition.GetJSON(ctx, kv, CapacityProviderKey(name), &existing)
 	if err != nil {
 		return nil, err
 	}
@@ -69,7 +70,7 @@ func (s *Service) CreateCapacityProvider(ctx context.Context, input *ecs.CreateC
 		Tags:                     tagsToMap(input.Tags),
 		CreatedAt:                time.Now().UTC(),
 	}
-	if err := putJSON(ctx, kv, CapacityProviderKey(name), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, CapacityProviderKey(name), &rec); err != nil {
 		return nil, err
 	}
 	return &ecs.CreateCapacityProviderOutput{CapacityProvider: rec.toAWS()}, nil
@@ -86,13 +87,13 @@ func (s *Service) DescribeCapacityProviders(ctx context.Context, input *ecs.Desc
 
 	names := awsStringSlice(input.CapacityProviders)
 	if len(names) == 0 {
-		keys, kerr := keysWithPrefix(ctx, kv, CapacityProvidersPrefix())
+		keys, kerr := taskdefinition.KeysWithPrefix(ctx, kv, CapacityProvidersPrefix())
 		if kerr != nil {
 			return nil, kerr
 		}
 		for _, k := range keys {
 			var rec CapacityProviderRecord
-			ok, gerr := getJSON(ctx, kv, k, &rec)
+			ok, gerr := taskdefinition.GetJSON(ctx, kv, k, &rec)
 			if gerr != nil {
 				return nil, gerr
 			}
@@ -106,7 +107,7 @@ func (s *Service) DescribeCapacityProviders(ctx context.Context, input *ecs.Desc
 	for _, ref := range names {
 		name := CapacityProviderShortName(ref)
 		var rec CapacityProviderRecord
-		ok, gerr := getJSON(ctx, kv, CapacityProviderKey(name), &rec)
+		ok, gerr := taskdefinition.GetJSON(ctx, kv, CapacityProviderKey(name), &rec)
 		if gerr != nil {
 			return nil, gerr
 		}
@@ -132,7 +133,7 @@ func (s *Service) DeleteCapacityProvider(ctx context.Context, input *ecs.DeleteC
 		return nil, err
 	}
 	var rec CapacityProviderRecord
-	found, err := getJSON(ctx, kv, CapacityProviderKey(name), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, CapacityProviderKey(name), &rec)
 	if err != nil {
 		return nil, err
 	}

spinifex/domains/ecs/cluster_teardown.go

@@ -3,6 +3,7 @@ package ecs
 import (
 	"context"
 	"errors"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -34,7 +35,7 @@ func (s *Service) DeleteCluster(ctx context.Context, input *ecs.DeleteClusterInp
 		return nil, err
 	}
 	var rec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -71,7 +72,7 @@ func (s *Service) DeregisterContainerInstance(ctx context.Context, input *ecs.De
 		return nil, err
 	}
 	var rec InstanceRecord
-	found, err := getJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -124,7 +125,7 @@ func (s *Service) UpdateContainerInstancesState(ctx context.Context, input *ecs.
 	for _, ref := range awsStringSlice(input.ContainerInstances) {
 		id := ContainerInstanceShortID(ref)
 		var rec InstanceRecord
-		found, gerr := getJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+		found, gerr := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 		if gerr != nil {
 			return nil, gerr
 		}
@@ -134,7 +135,7 @@ func (s *Service) UpdateContainerInstancesState(ctx context.Context, input *ecs.
 		}
 		rec.Status = status
 		rec.Reaped = false
-		if perr := putJSON(ctx, kv, InstanceKey(cluster, id), &rec); perr != nil {
+		if perr := taskdefinition.PutJSON(ctx, kv, InstanceKey(cluster, id), &rec); perr != nil {
 			return nil, perr
 		}
 		if status == InstanceStatusDraining {

spinifex/domains/ecs/cluster_teardown_test.go

@@ -28,7 +28,7 @@ func TestService_DeleteCluster_CascadesAndSweeps(t *testing.T) {
 	list, err := svc.ListClusters(context.Background(), &ecs.ListClustersInput{}, testAccountID)
 	require.NoError(t, err)
 	assert.Empty(t, list.ClusterArns)
-	keys, err := keysWithPrefix(t.Context(), kv, clusterKeyPrefix("web"))
+	keys, err := taskdefinition.KeysWithPrefix(t.Context(), kv, clusterKeyPrefix("web"))
 	require.NoError(t, err)
 	assert.Empty(t, keys)
 }

spinifex/domains/ecs/deployments_test.go

@@ -19,7 +19,7 @@ import (
 func reloadService(t *testing.T, kv jetstream.KeyValue, cluster, name string) *ServiceRecord {
 	t.Helper()
 	var rec ServiceRecord
-	found, err := getJSON(t.Context(), kv, ServiceKey(cluster, name), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, ServiceKey(cluster, name), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	return &rec
@@ -329,7 +329,7 @@ func TestDeployment_LegacyServiceSynthesizesPrimary(t *testing.T) {
 		DesiredCount: 1, Status: ServiceStatusActive,
 		SchedulingStrategy: SchedulingStrategyReplica, DeploymentID: "legacy-dep",
 	}
-	require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "legacy"), rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ServiceKey("web", "legacy"), rec))
 
 	require.NoError(t, svc.reconcileService(context.Background(), kv, testAccountID, rec))
 	reloaded := reloadService(t, kv, "web", "legacy")
@@ -388,7 +388,7 @@ func TestDeployment_RepeatedStartFailuresBackOffWithoutTheCircuitBreaker(t *test
 	// Once it passes, launching resumes: the backoff is a brake, not a stop.
 	rec = reloadService(t, kv, "web", "web")
 	rec.primaryDeployment().NextLaunchAt = time.Now().UTC().Add(-time.Second)
-	require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "web"), rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ServiceKey("web", "web"), rec))
 	require.NoError(t, svc.reconcileService(context.Background(), kv, testAccountID, reloadService(t, kv, "web", "web")))
 
 	assert.Positive(t, liveServiceTasks(t, svc, kv, "web", "web"), "backoff never released the deployment")

spinifex/domains/ecs/eip_test.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -37,14 +38,14 @@ func TestAssignTaskPublicIP_EnabledAssociatesAndPersists(t *testing.T) {
 	eips := &stubEIPs{publicIP: "192.0.2.50", allocID: "eipalloc-1"}
 	svc.eips = eips
 
-	require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{
 		Name: "web", Cluster: "web", AssignPublicIP: "ENABLED",
 	}))
 	task := &TaskRecord{
 		TaskID: "t-1", Cluster: "web", Group: serviceTaskGroup("web"),
 		ENIID: "eni-1", ENIPrivateIP: "172.31.0.8",
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), task))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), task))
 
 	svc.assignTaskPublicIP(context.Background(), kv, testAccountID, task)
 
@@ -53,7 +54,7 @@ func TestAssignTaskPublicIP_EnabledAssociatesAndPersists(t *testing.T) {
 	assert.Equal(t, "eipalloc-1", task.ENIEIPAllocationID)
 
 	var persisted TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &persisted)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &persisted)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, "192.0.2.50", persisted.ENIPublicIP)
@@ -64,7 +65,7 @@ func TestAssignTaskPublicIP_DisabledNoOp(t *testing.T) {
 	eips := &stubEIPs{publicIP: "192.0.2.50", allocID: "eipalloc-1"}
 	svc.eips = eips
 
-	require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{
 		Name: "web", Cluster: "web", // AssignPublicIP unset
 	}))
 	task := &TaskRecord{

spinifex/domains/ecs/eni_test.go

@@ -189,14 +189,14 @@ func singleTaskRecord(t *testing.T, svc *Service, cluster string) (TaskRecord, b
 	t.Helper()
 	kv, err := svc.bucket(context.Background(), testAccountID)
 	require.NoError(t, err)
-	keys, err := keysWithPrefix(context.Background(), kv, TasksPrefix(cluster))
+	keys, err := taskdefinition.KeysWithPrefix(context.Background(), kv, TasksPrefix(cluster))
 	require.NoError(t, err)
 	require.LessOrEqual(t, len(keys), 1, "test expects at most one task record")
 	if len(keys) == 0 {
 		return TaskRecord{}, false
 	}
 	var rec TaskRecord
-	found, err := getJSON(context.Background(), kv, keys[0], &rec)
+	found, err := taskdefinition.GetJSON(context.Background(), kv, keys[0], &rec)
 	require.NoError(t, err)
 	return rec, found
 }
@@ -317,10 +317,10 @@ func TestReaper_Awsvpc_ReleasesENI(t *testing.T) {
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	var rec InstanceRecord
-	_, err = getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
+	_, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
 	require.NoError(t, err)
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
 	sc := NewScheduler(nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())

spinifex/domains/ecs/export_test.go

@@ -15,8 +15,6 @@ const TestAccountID = testAccountID
 var (
 	NewTestService   = newTestService
 	RegisterInstance = registerInstance
-	PutJSON          = putJSON
-	GetJSON          = getJSON
 	ServiceTaskGroup = serviceTaskGroup
 )
 

spinifex/domains/ecs/gpu_report.go

@@ -1,6 +1,10 @@
 package ecs
 
-import "context"
+import (
+	"context"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
+)
 
 // ReportTaskGPUInput is the agent's per-task GPU device-assignment report
 // (agent → gateway). Internal agent↔gateway shape, not an AWS SDK action:
@@ -36,7 +40,7 @@ func (s *Service) ReportTaskGPU(ctx context.Context, input *ReportTaskGPUInput,
 		return nil, err
 	}
 	var task TaskRecord
-	found, err := getJSON(ctx, kv, TaskKey(cluster, taskID), &task)
+	found, err := taskdefinition.GetJSON(ctx, kv, TaskKey(cluster, taskID), &task)
 	if err != nil {
 		return nil, err
 	}
@@ -55,7 +59,7 @@ func (s *Service) ReportTaskGPU(ctx context.Context, input *ReportTaskGPUInput,
 		}
 	}
 	if changed {
-		if err := putJSON(ctx, kv, TaskKey(cluster, taskID), &task); err != nil {
+		if err := taskdefinition.PutJSON(ctx, kv, TaskKey(cluster, taskID), &task); err != nil {
 			return nil, err
 		}
 	}

spinifex/domains/ecs/gpu_report_test.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -17,7 +18,7 @@ func TestReportTaskGPU_MergesOntoTaskContainers(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", Cluster: "web",
 		Containers: []ContainerState{
 			{Name: "web", Status: "RUNNING"},
@@ -33,7 +34,7 @@ func TestReportTaskGPU_MergesOntoTaskContainers(t *testing.T) {
 	assert.Equal(t, "OK", out.Acknowledgment)
 
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	for _, c := range rec.Containers {
@@ -63,7 +64,7 @@ func TestDescribeTasks_GPUIDsPopulatedFromReport(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web",
 		Containers: []ContainerState{{Name: "trainer", Status: "RUNNING", GPUIDs: []string{"GPU-aaa", "GPU-bbb"}}},
 	}))
@@ -85,7 +86,7 @@ func TestRecordTaskState_PreservesGPUIDsAcrossStoppedReport(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web",
 		ContainerInstanceID: "i-1",
 		Containers:          []ContainerState{{Name: "trainer", Status: "PENDING"}},
@@ -99,7 +100,7 @@ func TestRecordTaskState_PreservesGPUIDsAcrossStoppedReport(t *testing.T) {
 	}))
 
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.Len(t, rec.Containers, 1)
@@ -114,7 +115,7 @@ func TestRecordTaskState_PreservesGPUIDsAcrossStoppedReport(t *testing.T) {
 		Containers: []ecsv1.ContainerStatus{{Name: "trainer", Status: ecsv1.TaskStatusStopped, ExitCode: &exit}},
 	}))
 
-	found, err = getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err = taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.Len(t, rec.Containers, 1)
@@ -137,7 +138,7 @@ func TestRecordTaskState_ReportedGPUIDsOverwritePinned(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web",
 		ContainerInstanceID: "i-1",
 		Containers:          []ContainerState{{Name: "trainer", Status: "RUNNING", GPUIDs: []string{"GPU-aaa"}}},
@@ -150,7 +151,7 @@ func TestRecordTaskState_ReportedGPUIDsOverwritePinned(t *testing.T) {
 	}))
 
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.Len(t, rec.Containers, 1)
@@ -163,7 +164,7 @@ func TestRecordTaskState_NonGPUContainerStaysEmpty(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web",
 		ContainerInstanceID: "i-1",
 		Containers:          []ContainerState{{Name: "web", Status: "PENDING"}},
@@ -183,7 +184,7 @@ func TestRecordTaskState_NonGPUContainerStaysEmpty(t *testing.T) {
 	}))
 
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.Len(t, rec.Containers, 1)

spinifex/domains/ecs/instance_recovery_test.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 	"time"
 
@@ -27,7 +28,7 @@ func instanceStatus(t *testing.T, svc *Service, cluster, id string) InstanceReco
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	var rec InstanceRecord
-	found, err := getJSON(t.Context(), kv, InstanceKey(cluster, id), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, InstanceKey(cluster, id), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	return rec
@@ -46,7 +47,7 @@ func TestRegister_RestoresReapedInstanceToActive(t *testing.T) {
 	require.NoError(t, err)
 	rec := instanceStatus(t, svc, "web", "i-1")
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
 	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
@@ -86,7 +87,7 @@ func TestReaper_ReleasesCapacityAndRecoversClean(t *testing.T) {
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	seeded.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &seeded))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &seeded))
 
 	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 

spinifex/domains/ecs/instances.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"log/slog"
 	"slices"
 	"time"
@@ -17,14 +18,14 @@ import (
 
 // listInstanceRecords returns all container-instance records in a cluster.
 func (s *Service) listInstanceRecords(ctx context.Context, kv jetstream.KeyValue, cluster string) ([]InstanceRecord, error) {
-	keys, err := keysWithPrefix(ctx, kv, InstancesPrefix(cluster))
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, InstancesPrefix(cluster))
 	if err != nil {
 		return nil, err
 	}
 	out := make([]InstanceRecord, 0, len(keys))
 	for _, k := range keys {
 		var rec InstanceRecord
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -138,7 +139,7 @@ func (s *Service) DescribeContainerInstances(ctx context.Context, input *ecs.Des
 	for _, ref := range awsStringSlice(input.ContainerInstances) {
 		id := ContainerInstanceShortID(ref)
 		var rec InstanceRecord
-		found, err := getJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -194,7 +195,7 @@ func (s *Service) ListContainerInstances(ctx context.Context, input *ecs.ListCon
 // it back. Used by both the AWS register path and the bus register handler.
 func (s *Service) upsertInstance(ctx context.Context, kv jetstream.KeyValue, accountID, cluster, instanceID string, mutate func(*InstanceRecord)) (*InstanceRecord, error) {
 	var rec InstanceRecord
-	found, err := getJSON(ctx, kv, InstanceKey(cluster, instanceID), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, instanceID), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -212,7 +213,7 @@ func (s *Service) upsertInstance(ctx context.Context, kv jetstream.KeyValue, acc
 	if mutate != nil {
 		mutate(&rec)
 	}
-	if err := putJSON(ctx, kv, InstanceKey(cluster, instanceID), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, InstanceKey(cluster, instanceID), &rec); err != nil {
 		return nil, err
 	}
 	return &rec, nil
@@ -345,7 +346,7 @@ func (s *Service) recordHeartbeat(ctx context.Context, msg *ecsv1.Heartbeat) err
 		return err
 	}
 	var rec InstanceRecord
-	found, err := getJSON(ctx, kv, InstanceKey(msg.ClusterName, msg.InstanceID), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, InstanceKey(msg.ClusterName, msg.InstanceID), &rec)
 	if err != nil || !found {
 		return err
 	}
@@ -353,7 +354,7 @@ func (s *Service) recordHeartbeat(ctx context.Context, msg *ecsv1.Heartbeat) err
 	if msg.Status != "" {
 		rec.Status = msg.Status
 	}
-	return putJSON(ctx, kv, InstanceKey(msg.ClusterName, msg.InstanceID), &rec)
+	return taskdefinition.PutJSON(ctx, kv, InstanceKey(msg.ClusterName, msg.InstanceID), &rec)
 }
 
 // recordTaskState applies an agent task-state report: it updates the task record
@@ -364,7 +365,7 @@ func (s *Service) recordTaskState(ctx context.Context, msg *ecsv1.TaskState) err
 		return err
 	}
 	var task TaskRecord
-	found, err := getJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task)
+	found, err := taskdefinition.GetJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task)
 	if err != nil || !found {
 		return err
 	}
@@ -406,7 +407,7 @@ func (s *Service) recordTaskState(ctx context.Context, msg *ecsv1.TaskState) err
 		// filtering on desired status would otherwise see it as still wanted.
 		task.DesiredStatus = TaskStatusStopped
 	}
-	if err := putJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task); err != nil {
 		return err
 	}
 
@@ -430,7 +431,7 @@ func (s *Service) recordTaskState(ctx context.Context, msg *ecsv1.TaskState) err
 		s.reclaimAssignInbox(ctx, kv, msg.ClusterName, task.ContainerInstanceID, msg.TaskID)
 		s.reclaimStopInbox(ctx, kv, msg.ClusterName, task.ContainerInstanceID, msg.TaskID)
 		s.reclaimTaskENI(ctx, msg.AccountID, &task)
-		if perr := putJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task); perr != nil {
+		if perr := taskdefinition.PutJSON(ctx, kv, TaskKey(msg.ClusterName, msg.TaskID), &task); perr != nil {
 			slog.ErrorContext(ctx, "ECS task STOPPED: persist after EIP release failed", "task", msg.TaskID, "err", perr)
 		}
 		return s.releaseReservation(ctx, kv, msg.ClusterName, task.ContainerInstanceID, msg.TaskID, task.ReservedCPU, task.ReservedMemoryMiB, task.GPU, eniReservationFor(task.NetworkMode))
@@ -485,7 +486,7 @@ func (s *Service) recordDeploymentFailure(ctx context.Context, kv jetstream.KeyV
 		return
 	}
 	var svc ServiceRecord
-	found, err := getJSON(ctx, kv, ServiceKey(cluster, name), &svc)
+	found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, name), &svc)
 	if err != nil || !found {
 		return
 	}
@@ -493,7 +494,7 @@ func (s *Service) recordDeploymentFailure(ctx context.Context, kv jetstream.KeyV
 		if svc.Deployments[i].ID == depID {
 			svc.Deployments[i].FailedTasks++
 			svc.Deployments[i].UpdatedAt = time.Now().UTC()
-			if perr := putJSON(ctx, kv, ServiceKey(cluster, name), &svc); perr != nil {
+			if perr := taskdefinition.PutJSON(ctx, kv, ServiceKey(cluster, name), &svc); perr != nil {
 				slog.ErrorContext(ctx, "ECS deployment failure accounting: persist failed", "service", name, "err", perr)
 			}
 			return

spinifex/domains/ecs/poll.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/nats-io/nats.go/jetstream"
@@ -55,14 +56,14 @@ func (s *Service) PollAssignments(ctx context.Context, input *PollAssignmentsInp
 		_ = kv.Delete(ctx, StopKey(cluster, instanceID, TaskShortID(taskID)))
 	}
 
-	keys, err := keysWithPrefix(ctx, kv, AssignmentsPrefix(cluster, instanceID))
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, AssignmentsPrefix(cluster, instanceID))
 	if err != nil {
 		return nil, err
 	}
 	out := &PollAssignmentsOutput{}
 	for _, key := range keys {
 		var as ecsv1.Assign
-		found, err := getJSON(ctx, kv, key, &as)
+		found, err := taskdefinition.GetJSON(ctx, kv, key, &as)
 		if err != nil {
 			return nil, err
 		}
@@ -71,13 +72,13 @@ func (s *Service) PollAssignments(ctx context.Context, input *PollAssignmentsInp
 		}
 	}
 
-	stopKeys, err := keysWithPrefix(ctx, kv, StopsPrefix(cluster, instanceID))
+	stopKeys, err := taskdefinition.KeysWithPrefix(ctx, kv, StopsPrefix(cluster, instanceID))
 	if err != nil {
 		return nil, err
 	}
 	for _, key := range stopKeys {
 		var sd ecsv1.StopDirective
-		found, err := getJSON(ctx, kv, key, &sd)
+		found, err := taskdefinition.GetJSON(ctx, kv, key, &sd)
 		if err != nil {
 			return nil, err
 		}

spinifex/domains/ecs/poll_test.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
@@ -16,9 +17,9 @@ import (
 func TestPollAssignments_DrainsStopsAndAcks(t *testing.T) {
 	svc, _, kv := serviceTestRig(t)
 
-	require.NoError(t, putJSON(t.Context(), kv, AssignmentKey("web", "i-1", "t-1"),
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, AssignmentKey("web", "i-1", "t-1"),
 		&ecsv1.Assign{TaskID: "t-1", ClusterName: "web", InstanceID: "i-1"}))
-	require.NoError(t, putJSON(t.Context(), kv, StopKey("web", "i-1", "t-2"),
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, StopKey("web", "i-1", "t-2"),
 		&ecsv1.StopDirective{TaskID: "t-2", Reason: "drain"}))
 
 	out, err := svc.PollAssignments(context.Background(), &PollAssignmentsInput{
@@ -41,7 +42,7 @@ func TestPollAssignments_DrainsStopsAndAcks(t *testing.T) {
 	assert.Empty(t, out.Assignments)
 	assert.Empty(t, out.Stops)
 
-	found, err := getJSON(t.Context(), kv, StopKey("web", "i-1", "t-2"), &ecsv1.StopDirective{})
+	found, err := taskdefinition.GetJSON(t.Context(), kv, StopKey("web", "i-1", "t-2"), &ecsv1.StopDirective{})
 	require.NoError(t, err)
 	assert.False(t, found)
 }

spinifex/domains/ecs/revisit_test.go

@@ -7,6 +7,7 @@ package ecs
 import (
 	"context"
 	"encoding/json"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 	"time"
 
@@ -60,11 +61,11 @@ func runOneTask(t *testing.T, svc *Service, kv jetstream.KeyValue) string {
 		Cluster: aws.String("web"), TaskDefinition: aws.String("app"), Count: aws.Int64(1),
 	}, testAccountID)
 	require.NoError(t, err)
-	keys, err := keysWithPrefix(t.Context(), kv, TasksPrefix("web"))
+	keys, err := taskdefinition.KeysWithPrefix(t.Context(), kv, TasksPrefix("web"))
 	require.NoError(t, err)
 	require.Len(t, keys, 1)
 	var task TaskRecord
-	found, err := getJSON(t.Context(), kv, keys[0], &task)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, keys[0], &task)
 	require.NoError(t, err)
 	require.True(t, found)
 	return task.TaskID
@@ -128,11 +129,11 @@ func TestScheduler_TheReaperAsksForTheSoonestHeartbeatWindow(t *testing.T) {
 	now := time.Now().UTC()
 
 	var inst InstanceRecord
-	found, err := getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst)
 	require.NoError(t, err)
 	require.True(t, found)
 	inst.LastSeen = now.Add(-30 * time.Second)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst))
 
 	due, err := sc.reap(t.Context())
 	require.NoError(t, err)
@@ -142,12 +143,12 @@ func TestScheduler_TheReaperAsksForTheSoonestHeartbeatWindow(t *testing.T) {
 	// Past the window the reaper acts, and with nothing live left it asks for
 	// nothing: the resync is the only thing that need bring it back.
 	inst.LastSeen = now.Add(-2 * heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst))
 	due, err = sc.reap(t.Context())
 	require.NoError(t, err)
 	assert.Zero(t, due)
 
-	found, err = getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst)
+	found, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, InstanceStatusDraining, inst.Status, "an instance past its window is still reaped")
@@ -195,17 +196,17 @@ func TestScheduler_TheTimerPassReturnsTheSoonestOfItsJobs(t *testing.T) {
 	// A live instance due in 10s, and a stopped task due much later: the pass must
 	// come back for the sooner of the two.
 	var inst InstanceRecord
-	found, err := getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst)
 	require.NoError(t, err)
 	require.True(t, found)
 	inst.LastSeen = now.Add(10*time.Second - heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &inst))
 
 	stopped := TaskRecord{
 		TaskID: "t-1", Cluster: "web", ARN: TaskARN(testRegion, testAccountID, "web", "t-1"),
 		LastStatus: TaskStatusStopped, DesiredStatus: TaskStatusStopped, StoppedAt: now,
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &stopped))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &stopped))
 
 	reapDue, err := sc.reap(t.Context())
 	require.NoError(t, err)
@@ -231,11 +232,11 @@ func TestScheduler_AnInFlightServiceAsksToBeRevisited(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, reconcileInterval, due)
 
-	keys, err := keysWithPrefix(t.Context(), kv, TasksPrefix("web"))
+	keys, err := taskdefinition.KeysWithPrefix(t.Context(), kv, TasksPrefix("web"))
 	require.NoError(t, err)
 	require.Len(t, keys, 1)
 	var task TaskRecord
-	found, err := getJSON(t.Context(), kv, keys[0], &task)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, keys[0], &task)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.NoError(t, svc.recordTaskState(t.Context(), &ecsv1.TaskState{

spinifex/domains/ecs/scheduler.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"log/slog"
 	"strings"
 	"sync"
@@ -362,7 +363,7 @@ func (sc *Scheduler) reap(ctx context.Context) (time.Duration, error) {
 // reapBucket reaps one account's disconnected instances and reports when the
 // soonest instance it left alone falls due.
 func (sc *Scheduler) reapBucket(ctx context.Context, kv jetstream.KeyValue, accountID string, now time.Time) time.Duration {
-	keys, err := keysWithPrefix(ctx, kv, "clusters/")
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, "clusters/")
 	if err != nil {
 		return 0
 	}
@@ -372,7 +373,7 @@ func (sc *Scheduler) reapBucket(ctx context.Context, kv jetstream.KeyValue, acco
 			continue
 		}
 		var inst InstanceRecord
-		found, err := getJSON(ctx, kv, k, &inst)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &inst)
 		if err != nil || !found {
 			continue
 		}
@@ -387,7 +388,7 @@ func (sc *Scheduler) reapBucket(ctx context.Context, kv jetstream.KeyValue, acco
 			"lastSeen", inst.LastSeen.Format(time.RFC3339))
 		inst.Status = InstanceStatusDraining
 		inst.Reaped = true
-		if perr := putJSON(ctx, kv, k, &inst); perr != nil {
+		if perr := taskdefinition.PutJSON(ctx, kv, k, &inst); perr != nil {
 			continue
 		}
 		sc.stopInstanceTasks(ctx, kv, accountID, inst.Cluster, inst.InstanceID)
@@ -398,13 +399,13 @@ func (sc *Scheduler) reapBucket(ctx context.Context, kv jetstream.KeyValue, acco
 // stopInstanceTasks transitions a reaped instance's non-stopped tasks to STOPPED
 // and reclaims each awsvpc task's ENI (leak guard for a dead agent).
 func (sc *Scheduler) stopInstanceTasks(ctx context.Context, kv jetstream.KeyValue, accountID, cluster, instanceID string) {
-	keys, err := keysWithPrefix(ctx, kv, TasksPrefix(cluster))
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, TasksPrefix(cluster))
 	if err != nil {
 		return
 	}
 	for _, k := range keys {
 		var task TaskRecord
-		found, err := getJSON(ctx, kv, k, &task)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &task)
 		if err != nil || !found {
 			continue
 		}

spinifex/domains/ecs/service.go

@@ -5,9 +5,7 @@ package ecs
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
-	"fmt"
 	"log/slog"
 	"maps"
 	"slices"
@@ -168,39 +166,12 @@ func (s *Service) bucket(ctx context.Context, accountID string) (jetstream.KeyVa
 	return GetOrCreateAccountBucket(ctx, js, accountID)
 }
 
-// getJSON reads key into out. Returns (false, nil) when the key is absent.
-func getJSON(ctx context.Context, kv jetstream.KeyValue, key string, out any) (bool, error) {
-	entry, err := kv.Get(ctx, key)
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return false, nil
-	}
-	if err != nil {
-		return false, err
-	}
-	if err := json.Unmarshal(entry.Value(), out); err != nil {
-		return false, fmt.Errorf("unmarshal %s: %w", key, err)
-	}
-	return true, nil
-}
-
-// putJSON marshals v and writes it at key.
-func putJSON(ctx context.Context, kv jetstream.KeyValue, key string, v any) error {
-	data, err := json.Marshal(v)
-	if err != nil {
-		return err
-	}
-	if _, err := kv.Put(ctx, key, data); err != nil {
-		return fmt.Errorf("put %s: %w", key, err)
-	}
-	return nil
-}
-
 // deleteKeysWithPrefix deletes every KV key under prefix. Used by DeleteCluster
 // to sweep a cluster's whole key subtree (meta/instances/tasks/services/
 // assignments) after its tasks are stopped. Best-effort per key; the first
 // delete error is returned after attempting the rest.
 func deleteKeysWithPrefix(ctx context.Context, kv jetstream.KeyValue, prefix string) error {
-	keys, err := keysWithPrefix(ctx, kv, prefix)
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, prefix)
 	if err != nil {
 		return err
 	}
@@ -213,25 +184,6 @@ func deleteKeysWithPrefix(ctx context.Context, kv jetstream.KeyValue, prefix str
 	return firstErr
 }
 
-// keysWithPrefix returns all KV keys under prefix.
-func keysWithPrefix(ctx context.Context, kv jetstream.KeyValue, prefix string) ([]string, error) {
-	keys, err := kv.Keys(ctx)
-	if errors.Is(err, jetstream.ErrNoKeysFound) {
-		return nil, nil
-	}
-	if err != nil {
-		return nil, err
-	}
-	out := make([]string, 0, len(keys))
-	for _, k := range keys {
-		if strings.HasPrefix(k, prefix) {
-			out = append(out, k)
-		}
-	}
-	slices.Sort(out)
-	return out, nil
-}
-
 // --- Cluster ---
 
 // CreateCluster persists a cluster meta record. Idempotent: re-creating an
@@ -247,7 +199,7 @@ func (s *Service) CreateCluster(ctx context.Context, input *ecs.CreateClusterInp
 	}
 
 	var rec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(name), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(name), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -269,7 +221,7 @@ func (s *Service) CreateCluster(ctx context.Context, input *ecs.CreateClusterInp
 				rec.Tags[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
 			}
 		}
-		if err := putJSON(ctx, kv, ClusterMetaKey(name), &rec); err != nil {
+		if err := taskdefinition.PutJSON(ctx, kv, ClusterMetaKey(name), &rec); err != nil {
 			return nil, err
 		}
 	}
@@ -292,7 +244,7 @@ func (s *Service) DescribeClusters(ctx context.Context, input *ecs.DescribeClust
 	for _, name := range names {
 		name = ClusterShortName(name)
 		var rec ClusterRecord
-		found, err := getJSON(ctx, kv, ClusterMetaKey(name), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(name), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -358,7 +310,7 @@ func (s *Service) ListClusters(ctx context.Context, _ *ecs.ListClustersInput, ac
 	if err != nil {
 		return nil, err
 	}
-	keys, err := keysWithPrefix(ctx, kv, "clusters/")
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, "clusters/")
 	if err != nil {
 		return nil, err
 	}
@@ -368,7 +320,7 @@ func (s *Service) ListClusters(ctx context.Context, _ *ecs.ListClustersInput, ac
 			continue
 		}
 		var rec ClusterRecord
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}

spinifex/domains/ecs/service_test.go

@@ -378,7 +378,7 @@ func TestService_TaskCredIDIsMintedSecret(t *testing.T) {
 		assert.Equal(t, as.CredID, second[taskID].CredID, "re-delivered assign changed credential ID")
 
 		var rec TaskRecord
-		found, gerr := getJSON(ctx, kv, TaskKey("web", taskID), &rec)
+		found, gerr := taskdefinition.GetJSON(ctx, kv, TaskKey("web", taskID), &rec)
 		require.NoError(t, gerr)
 		require.True(t, found)
 		assert.Equal(t, as.CredID, rec.CredID)
@@ -558,10 +558,10 @@ func TestScheduler_ReapBucket_StopsStaleInstanceTasks(t *testing.T) {
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	var rec InstanceRecord
-	_, err = getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
+	_, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
 	require.NoError(t, err)
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
 	sc := NewScheduler(nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
@@ -721,7 +721,7 @@ func TestService_RunTask_GPU_ReservesOnPlacement(t *testing.T) {
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	var rec InstanceRecord
-	found, err := getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, 2, rec.TotalGPU)
@@ -732,7 +732,7 @@ func TestService_RunTask_GPU_ReservesOnPlacement(t *testing.T) {
 		AccountID: testAccountID, ClusterName: "web", InstanceID: "i-1", TaskID: taskID,
 		LastStatus: ecsv1.TaskStatusStopped, Reason: "exited",
 	}))
-	found, err = getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
+	found, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, 0, rec.ReservedGPU)

spinifex/domains/ecs/services.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"log/slog"
 	"strings"
 	"time"
@@ -145,7 +146,7 @@ func (s *Service) CreateService(ctx context.Context, input *ecs.CreateServiceInp
 		return nil, err
 	}
 	var clusterRec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
 	if err != nil {
 		return nil, err
 	}
@@ -154,7 +155,7 @@ func (s *Service) CreateService(ctx context.Context, input *ecs.CreateServiceInp
 	}
 
 	var existing ServiceRecord
-	if ok, gerr := getJSON(ctx, kv, ServiceKey(cluster, name), &existing); gerr != nil {
+	if ok, gerr := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, name), &existing); gerr != nil {
 		return nil, gerr
 	} else if ok && existing.Status == ServiceStatusActive {
 		return &ecs.CreateServiceOutput{Service: s.serviceToAWS(accountID, &existing)}, nil
@@ -195,7 +196,7 @@ func (s *Service) CreateService(ctx context.Context, input *ecs.CreateServiceInp
 	applyDeploymentConfig(&rec, input.DeploymentConfiguration)
 	rec.LastGoodTaskDefARN = taskDef.ARN
 	rec.Deployments = []Deployment{newPrimaryDeployment(rec.DeploymentID, taskDef, rec.DesiredCount)}
-	if err := putJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil {
 		return nil, err
 	}
 
@@ -214,7 +215,7 @@ func (s *Service) UpdateService(ctx context.Context, input *ecs.UpdateServiceInp
 		return nil, err
 	}
 	var rec ServiceRecord
-	found, err := getJSON(ctx, kv, ServiceKey(cluster, name), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, name), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -253,7 +254,7 @@ func (s *Service) UpdateService(ctx context.Context, input *ecs.UpdateServiceInp
 		}
 	}
 	rec.UpdatedAt = time.Now().UTC()
-	if err := putJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil {
 		return nil, err
 	}
 	if err := s.reconcileService(ctx, kv, accountID, &rec); err != nil {
@@ -272,7 +273,7 @@ func (s *Service) DeleteService(ctx context.Context, input *ecs.DeleteServiceInp
 		return nil, err
 	}
 	var rec ServiceRecord
-	found, err := getJSON(ctx, kv, ServiceKey(cluster, name), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, name), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -292,7 +293,7 @@ func (s *Service) DeleteService(ctx context.Context, input *ecs.DeleteServiceInp
 	rec.RunningCount = 0
 	rec.PendingCount = 0
 	rec.UpdatedAt = time.Now().UTC()
-	if err := putJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, ServiceKey(cluster, name), &rec); err != nil {
 		return nil, err
 	}
 	return &ecs.DeleteServiceOutput{Service: s.serviceToAWS(accountID, &rec)}, nil
@@ -309,7 +310,7 @@ func (s *Service) DescribeServices(ctx context.Context, input *ecs.DescribeServi
 	for _, ref := range awsStringSlice(input.Services) {
 		name := ServiceShortName(ref)
 		var rec ServiceRecord
-		found, gerr := getJSON(ctx, kv, ServiceKey(cluster, name), &rec)
+		found, gerr := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, name), &rec)
 		if gerr != nil {
 			return nil, gerr
 		}
@@ -430,7 +431,7 @@ func persistServiceIfChanged(ctx context.Context, kv jetstream.KeyValue, svc *Se
 	if bytes.Equal(before, after) {
 		return nil
 	}
-	return putJSON(ctx, kv, ServiceKey(svc.Cluster, svc.Name), svc)
+	return taskdefinition.PutJSON(ctx, kv, ServiceKey(svc.Cluster, svc.Name), svc)
 }
 
 // reconcileAllServices reconciles every ACTIVE service in every ECS account
@@ -603,14 +604,14 @@ func (s *Service) listServiceTasks(ctx context.Context, kv jetstream.KeyValue, c
 
 // listTaskRecords returns every task record in a cluster.
 func (s *Service) listTaskRecords(ctx context.Context, kv jetstream.KeyValue, cluster string) ([]TaskRecord, error) {
-	keys, err := keysWithPrefix(ctx, kv, TasksPrefix(cluster))
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, TasksPrefix(cluster))
 	if err != nil {
 		return nil, err
 	}
 	out := make([]TaskRecord, 0, len(keys))
 	for _, k := range keys {
 		var rec TaskRecord
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -623,14 +624,14 @@ func (s *Service) listTaskRecords(ctx context.Context, kv jetstream.KeyValue, cl
 
 // listServiceRecords returns every service record in a cluster.
 func (s *Service) listServiceRecords(ctx context.Context, kv jetstream.KeyValue, cluster string) ([]ServiceRecord, error) {
-	keys, err := keysWithPrefix(ctx, kv, ServicesPrefix(cluster))
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, ServicesPrefix(cluster))
 	if err != nil {
 		return nil, err
 	}
 	out := make([]ServiceRecord, 0, len(keys))
 	for _, k := range keys {
 		var rec ServiceRecord
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -643,7 +644,7 @@ func (s *Service) listServiceRecords(ctx context.Context, kv jetstream.KeyValue,
 
 // allServiceRecords returns every service record across all clusters in a bucket.
 func (s *Service) allServiceRecords(ctx context.Context, kv jetstream.KeyValue) ([]ServiceRecord, error) {
-	keys, err := keysWithPrefix(ctx, kv, "clusters/")
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, "clusters/")
 	if err != nil {
 		return nil, err
 	}
@@ -653,7 +654,7 @@ func (s *Service) allServiceRecords(ctx context.Context, kv jetstream.KeyValue)
 			continue
 		}
 		var rec ServiceRecord
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}

spinifex/domains/ecs/services_test.go

@@ -3,6 +3,7 @@ package ecs
 import (
 	"context"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 	"time"
 
@@ -469,7 +470,7 @@ func TestService_StopTask_StopsAndDeregisters(t *testing.T) {
 		NetworkMode: NetworkModeAwsvpc, ENIPrivateIP: "10.0.1.9",
 		ReservedCPU: 64, ReservedMemoryMiB: 64,
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-9"), task))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-9"), task))
 
 	// Cooperative stop: the task stays RUNNING with desiredStatus=STOPPED and a
 	// stop directive lands in the instance inbox; nothing is deregistered until the
@@ -483,7 +484,7 @@ func TestService_StopTask_StopsAndDeregisters(t *testing.T) {
 	assert.Empty(t, reg.deregistered)
 
 	var sd ecsv1.StopDirective
-	found, err := getJSON(t.Context(), kv, StopKey("web", "i-1", "t-9"), &sd)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, StopKey("web", "i-1", "t-9"), &sd)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, "bye", sd.Reason)
@@ -494,7 +495,7 @@ func TestService_StopTask_StopsAndDeregisters(t *testing.T) {
 		TaskID: "t-9", LastStatus: ecsv1.TaskStatusStopped,
 	}))
 	assert.Equal(t, []string{key(tgARN, "10.0.1.9", 80)}, reg.deregistered)
-	found, err = getJSON(t.Context(), kv, StopKey("web", "i-1", "t-9"), &sd)
+	found, err = taskdefinition.GetJSON(t.Context(), kv, StopKey("web", "i-1", "t-9"), &sd)
 	require.NoError(t, err)
 	assert.False(t, found)
 
@@ -518,7 +519,7 @@ func TestService_StopTask_UnplacedForceStops(t *testing.T) {
 		TaskID: "t-np", Cluster: "web",
 		LastStatus: TaskStatusPending, DesiredStatus: TaskStatusRunning,
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-np"), task))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-np"), task))
 
 	out, err := svc.StopTask(context.Background(), &ecs.StopTaskInput{
 		Cluster: aws.String("web"), Task: aws.String("t-np"), Reason: aws.String("cancel"),
@@ -550,7 +551,7 @@ func TestService_TargetGroup_RegisterOnRunningDeregisterOnStopped(t *testing.T)
 		ContainerInstanceID: "i-1", LastStatus: TaskStatusPending,
 		NetworkMode: NetworkModeAwsvpc, ENIPrivateIP: "10.0.1.5",
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), task))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), task))
 
 	// RUNNING → register.
 	require.NoError(t, svc.recordTaskState(context.Background(), &ecsv1.TaskState{

spinifex/domains/ecs/store.go

@@ -107,12 +107,6 @@ func ServiceKey(cluster, serviceName string) string {
 	return ServicesPrefix(cluster) + serviceName
 }
 
-// LeaderLeaseKey returns the per-cluster scheduler leader-lease key in the shared
-// leader bucket (Q3).
-func LeaderLeaseKey(accountID, clusterName string) string {
-	return fmt.Sprintf("%s/%s", accountID, clusterName)
-}
-
 // CapacityProvidersPrefix returns the KV key prefix under which all
 // capacity-provider records live. Account-scoped (not cluster-scoped),
 // matching the AWS ARN shape; a cluster references providers by name.

spinifex/domains/ecs/store_test.go

@@ -124,8 +124,6 @@ func TestKeyPaths(t *testing.T) {
 
 	assert.Equal(t, "clusters/web/services/", ServicesPrefix("web"))
 	assert.Equal(t, "clusters/web/services/api", ServiceKey("web", "api"))
-
-	assert.Equal(t, "123456789012/web", LeaderLeaseKey("123456789012", "web"))
 }
 
 // Prefix helpers must be a true prefix of their per-record key so a KV

spinifex/domains/ecs/tagging_current_behaviour_test.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"strings"
 	"sync"
 	"testing"
@@ -57,11 +58,11 @@ func TestCurrentBehaviour_Tagging_ECSTagWriteRevertsInterleavedRecordWrite(t *te
 	key := ServiceKey("web", "api")
 	wrapped := &interleavingKV{KeyValue: kv, key: key, between: func() {
 		var rec ServiceRecord
-		found, gerr := getJSON(ctx, kv, key, &rec)
+		found, gerr := taskdefinition.GetJSON(ctx, kv, key, &rec)
 		require.NoError(t, gerr)
 		require.True(t, found)
 		rec.DesiredCount = 5
-		require.NoError(t, putJSON(ctx, kv, key, &rec))
+		require.NoError(t, taskdefinition.PutJSON(ctx, kv, key, &rec))
 	}}
 
 	err = svc.mutateResourceTags(ctx, wrapped, ecsResourceService, "web", "api", func(existing map[string]string) map[string]string {
@@ -74,7 +75,7 @@ func TestCurrentBehaviour_Tagging_ECSTagWriteRevertsInterleavedRecordWrite(t *te
 	require.NoError(t, err)
 
 	var final ServiceRecord
-	found, err := getJSON(ctx, kv, key, &final)
+	found, err := taskdefinition.GetJSON(ctx, kv, key, &final)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, 0, final.DesiredCount, "the interleaved DesiredCount=5 write is lost")

spinifex/domains/ecs/tags.go

@@ -78,7 +78,7 @@ func (s *Service) resourceTags(ctx context.Context, kv jetstream.KeyValue, kind
 	switch kind {
 	case ecsResourceCluster:
 		var rec ClusterRecord
-		found, err := getJSON(ctx, kv, ClusterMetaKey(id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(id), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -92,7 +92,7 @@ func (s *Service) resourceTags(ctx context.Context, kv jetstream.KeyValue, kind
 			return nil, errors.New(awserrors.ErrorECSInvalidParameter)
 		}
 		var rec taskdefinition.Record
-		found, err := getJSON(ctx, kv, taskdefinition.RevKey(family, rev), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, taskdefinition.RevKey(family, rev), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -102,7 +102,7 @@ func (s *Service) resourceTags(ctx context.Context, kv jetstream.KeyValue, kind
 		return rec.Tags, nil
 	case ecsResourceService:
 		var rec ServiceRecord
-		found, err := getJSON(ctx, kv, ServiceKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, id), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -112,7 +112,7 @@ func (s *Service) resourceTags(ctx context.Context, kv jetstream.KeyValue, kind
 		return rec.Tags, nil
 	case ecsResourceTask:
 		var rec TaskRecord
-		found, err := getJSON(ctx, kv, TaskKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, TaskKey(cluster, id), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -122,7 +122,7 @@ func (s *Service) resourceTags(ctx context.Context, kv jetstream.KeyValue, kind
 		return rec.Tags, nil
 	case ecsResourceContainerInstance:
 		var rec InstanceRecord
-		found, err := getJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -142,7 +142,7 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 	switch kind {
 	case ecsResourceCluster:
 		var rec ClusterRecord
-		found, err := getJSON(ctx, kv, ClusterMetaKey(id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(id), &rec)
 		if err != nil {
 			return err
 		}
@@ -150,14 +150,14 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 			return errors.New(awserrors.ErrorECSClusterNotFound)
 		}
 		rec.Tags = mutate(rec.Tags)
-		return putJSON(ctx, kv, ClusterMetaKey(id), &rec)
+		return taskdefinition.PutJSON(ctx, kv, ClusterMetaKey(id), &rec)
 	case ecsResourceTaskDefinition:
 		family, rev := taskdefinition.ParseRef(id)
 		if family == "" || rev == 0 {
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}
 		var rec taskdefinition.Record
-		found, err := getJSON(ctx, kv, taskdefinition.RevKey(family, rev), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, taskdefinition.RevKey(family, rev), &rec)
 		if err != nil {
 			return err
 		}
@@ -165,10 +165,10 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}
 		rec.Tags = mutate(rec.Tags)
-		return putJSON(ctx, kv, taskdefinition.RevKey(family, rev), &rec)
+		return taskdefinition.PutJSON(ctx, kv, taskdefinition.RevKey(family, rev), &rec)
 	case ecsResourceService:
 		var rec ServiceRecord
-		found, err := getJSON(ctx, kv, ServiceKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(cluster, id), &rec)
 		if err != nil {
 			return err
 		}
@@ -176,10 +176,10 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 			return errors.New(awserrors.ErrorECSServiceNotFound)
 		}
 		rec.Tags = mutate(rec.Tags)
-		return putJSON(ctx, kv, ServiceKey(cluster, id), &rec)
+		return taskdefinition.PutJSON(ctx, kv, ServiceKey(cluster, id), &rec)
 	case ecsResourceTask:
 		var rec TaskRecord
-		found, err := getJSON(ctx, kv, TaskKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, TaskKey(cluster, id), &rec)
 		if err != nil {
 			return err
 		}
@@ -187,10 +187,10 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}
 		rec.Tags = mutate(rec.Tags)
-		return putJSON(ctx, kv, TaskKey(cluster, id), &rec)
+		return taskdefinition.PutJSON(ctx, kv, TaskKey(cluster, id), &rec)
 	case ecsResourceContainerInstance:
 		var rec InstanceRecord
-		found, err := getJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 		if err != nil {
 			return err
 		}
@@ -198,7 +198,7 @@ func (s *Service) mutateResourceTags(ctx context.Context, kv jetstream.KeyValue,
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}
 		rec.Tags = mutate(rec.Tags)
-		return putJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+		return taskdefinition.PutJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 	default:
 		return errors.New(awserrors.ErrorECSInvalidParameter)
 	}

spinifex/domains/ecs/task_accounting_test.go

@@ -1,6 +1,7 @@
 package ecs_test
 
 import (
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -23,7 +24,7 @@ func seedTasks(t *testing.T, svc *ecsdomain.Service, cluster string, recs ...*ec
 	for _, r := range recs {
 		r.Cluster = cluster
 		r.ARN = ecsdomain.TaskARN(svc.Region(), ecsdomain.TestAccountID, cluster, r.TaskID)
-		require.NoError(t, ecsdomain.PutJSON(t.Context(), kv, ecsdomain.TaskKey(cluster, r.TaskID), r))
+		require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ecsdomain.TaskKey(cluster, r.TaskID), r))
 	}
 }
 
@@ -124,7 +125,7 @@ func TestRecordTaskState_SelfExitMovesDesiredStatusToStopped(t *testing.T) {
 	kv, err := svc.Bucket(t.Context(), ecsdomain.TestAccountID)
 	require.NoError(t, err)
 	var rec ecsdomain.TaskRecord
-	found, err := ecsdomain.GetJSON(t.Context(), kv, ecsdomain.TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, ecsdomain.TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, ecsdomain.TaskStatusStopped, rec.DesiredStatus)

spinifex/domains/ecs/task_stop.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"log/slog"
 	"strings"
 	"time"
@@ -33,7 +34,7 @@ func (s *Service) StopTask(ctx context.Context, input *ecs.StopTaskInput, accoun
 		return nil, err
 	}
 	var task TaskRecord
-	found, err := getJSON(ctx, kv, TaskKey(cluster, taskID), &task)
+	found, err := taskdefinition.GetJSON(ctx, kv, TaskKey(cluster, taskID), &task)
 	if err != nil {
 		return nil, err
 	}
@@ -57,7 +58,7 @@ func (s *Service) StartTask(ctx context.Context, input *ecs.StartTaskInput, acco
 		return nil, err
 	}
 	var clusterRec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
 	if err != nil {
 		return nil, err
 	}
@@ -101,7 +102,7 @@ func (s *Service) StartTask(ctx context.Context, input *ecs.StartTaskInput, acco
 				continue
 			}
 		}
-		if err := putJSON(ctx, kv, TaskKey(cluster, taskID), rec); err != nil {
+		if err := taskdefinition.PutJSON(ctx, kv, TaskKey(cluster, taskID), rec); err != nil {
 			return nil, err
 		}
 		if err := s.publishAssign(ctx, kv, accountID, cluster, inst.InstanceID, rec, taskDef); err != nil {
@@ -168,12 +169,12 @@ func (s *Service) requestStopTask(ctx context.Context, kv jetstream.KeyValue, ac
 		return
 	}
 	task.DesiredStatus = TaskStatusStopped
-	if perr := putJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil {
+	if perr := taskdefinition.PutJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil {
 		slog.ErrorContext(ctx, "ECS requestStopTask: persist failed", "task", task.TaskID, "err", perr)
 		return
 	}
 	sd := ecsv1.StopDirective{TaskID: task.TaskID, Reason: reason}
-	if perr := putJSON(ctx, kv, StopKey(task.Cluster, task.ContainerInstanceID, task.TaskID), &sd); perr != nil {
+	if perr := taskdefinition.PutJSON(ctx, kv, StopKey(task.Cluster, task.ContainerInstanceID, task.TaskID), &sd); perr != nil {
 		slog.ErrorContext(ctx, "ECS requestStopTask: post stop directive failed", "task", task.TaskID, "err", perr)
 	}
 }
@@ -196,7 +197,7 @@ func (s *Service) forceStopTask(ctx context.Context, kv jetstream.KeyValue, acco
 	// cleared fields land in this single write.
 	s.releaseTaskPublicIP(ctx, accountID, task)
 	s.reclaimTaskENI(ctx, accountID, task)
-	if perr := putJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil {
+	if perr := taskdefinition.PutJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil {
 		slog.ErrorContext(ctx, "ECS forceStopTask: persist failed", "task", task.TaskID, "err", perr)
 		return
 	}
@@ -238,7 +239,7 @@ func (s *Service) applyServiceTargets(ctx context.Context, kv jetstream.KeyValue
 		return
 	}
 	var svc ServiceRecord
-	found, err := getJSON(ctx, kv, ServiceKey(task.Cluster, name), &svc)
+	found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(task.Cluster, name), &svc)
 	if err != nil || !found || len(svc.LoadBalancers) == 0 {
 		return
 	}
@@ -270,7 +271,7 @@ func (s *Service) assignTaskPublicIP(ctx context.Context, kv jetstream.KeyValue,
 		return
 	}
 	var svc ServiceRecord
-	found, err := getJSON(ctx, kv, ServiceKey(task.Cluster, name), &svc)
+	found, err := taskdefinition.GetJSON(ctx, kv, ServiceKey(task.Cluster, name), &svc)
 	if err != nil || !found || !strings.EqualFold(svc.AssignPublicIP, "ENABLED") {
 		return
 	}
@@ -281,7 +282,7 @@ func (s *Service) assignTaskPublicIP(ctx context.Context, kv jetstream.KeyValue,
 	}
 	task.ENIPublicIP = publicIP
 	task.ENIEIPAllocationID = allocID
-	if perr := putJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil {
+	if perr := taskdefinition.PutJSON(ctx, kv, TaskKey(task.Cluster, task.TaskID), task); perr != nil {
 		slog.ErrorContext(ctx, "ECS auto-EIP: persist public IP failed", "task", task.TaskID, "err", perr)
 	}
 }

spinifex/domains/ecs/taskdef_validation_test.go

@@ -3,6 +3,7 @@ package ecs
 import (
 	"context"
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -240,7 +241,7 @@ func TestRunTask_AssignCarriesGPU(t *testing.T) {
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", taskID), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", taskID), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, 1, rec.GPU)

spinifex/domains/ecs/taskdefinition/owner.go

@@ -63,10 +63,10 @@ func (o *Owner) Register(ctx context.Context, kv jetstream.KeyValue, accountID s
 		RequiresCompatibilities: spec.RequiresCompatibilities,
 		RuntimePlatform:         spec.RuntimePlatform,
 	}
-	if err := putJSON(ctx, kv, RevKey(spec.Family, rev), rec); err != nil {
+	if err := PutJSON(ctx, kv, RevKey(spec.Family, rev), rec); err != nil {
 		return nil, err
 	}
-	if err := putJSON(ctx, kv, LatestRevKey(spec.Family), rev); err != nil {
+	if err := PutJSON(ctx, kv, LatestRevKey(spec.Family), rev); err != nil {
 		return nil, err
 	}
 	return rec, nil
@@ -75,7 +75,7 @@ func (o *Owner) Register(ctx context.Context, kv jetstream.KeyValue, accountID s
 // nextRevision reads the family's latest-rev and returns latest+1 (1 if absent).
 func nextRevision(ctx context.Context, kv jetstream.KeyValue, family string) (int, error) {
 	var latest int
-	found, err := getJSON(ctx, kv, LatestRevKey(family), &latest)
+	found, err := GetJSON(ctx, kv, LatestRevKey(family), &latest)
 	if err != nil {
 		return 0, err
 	}
@@ -88,7 +88,7 @@ func nextRevision(ctx context.Context, kv jetstream.KeyValue, family string) (in
 // Deregister marks one revision INACTIVE; the revision stays describable.
 func (o *Owner) Deregister(ctx context.Context, kv jetstream.KeyValue, family string, rev int) (*Record, error) {
 	var rec Record
-	found, err := getJSON(ctx, kv, RevKey(family, rev), &rec)
+	found, err := GetJSON(ctx, kv, RevKey(family, rev), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -100,7 +100,7 @@ func (o *Owner) Deregister(ctx context.Context, kv jetstream.KeyValue, family st
 		rec.DeregisteredAt = &now
 	}
 	rec.Status = StatusInactive
-	if err := putJSON(ctx, kv, RevKey(family, rev), &rec); err != nil {
+	if err := PutJSON(ctx, kv, RevKey(family, rev), &rec); err != nil {
 		return nil, err
 	}
 	return &rec, nil
@@ -115,7 +115,7 @@ func (o *Owner) Resolve(ctx context.Context, kv jetstream.KeyValue, ref string)
 	}
 	if rev == 0 {
 		var latest int
-		found, err := getJSON(ctx, kv, LatestRevKey(family), &latest)
+		found, err := GetJSON(ctx, kv, LatestRevKey(family), &latest)
 		if err != nil {
 			return nil, err
 		}
@@ -125,7 +125,7 @@ func (o *Owner) Resolve(ctx context.Context, kv jetstream.KeyValue, ref string)
 		rev = latest
 	}
 	var rec Record
-	found, err := getJSON(ctx, kv, RevKey(family, rev), &rec)
+	found, err := GetJSON(ctx, kv, RevKey(family, rev), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -142,7 +142,7 @@ func (o *Owner) List(ctx context.Context, kv jetstream.KeyValue, family, status
 	if family != "" {
 		prefix = RevsPrefix(family)
 	}
-	keys, err := keysWithPrefix(ctx, kv, prefix)
+	keys, err := KeysWithPrefix(ctx, kv, prefix)
 	if err != nil {
 		return nil, err
 	}
@@ -152,7 +152,7 @@ func (o *Owner) List(ctx context.Context, kv jetstream.KeyValue, family, status
 			continue
 		}
 		var rec Record
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -163,8 +163,8 @@ func (o *Owner) List(ctx context.Context, kv jetstream.KeyValue, family, status
 	return arns, nil
 }
 
-// getJSON reads key into out. Returns (false, nil) when the key is absent.
-func getJSON(ctx context.Context, kv jetstream.KeyValue, key string, out any) (bool, error) {
+// GetJSON reads key into out. Returns (false, nil) when the key is absent.
+func GetJSON(ctx context.Context, kv jetstream.KeyValue, key string, out any) (bool, error) {
 	entry, err := kv.Get(ctx, key)
 	if errors.Is(err, jetstream.ErrKeyNotFound) {
 		return false, nil
@@ -178,8 +178,8 @@ func getJSON(ctx context.Context, kv jetstream.KeyValue, key string, out any) (b
 	return true, nil
 }
 
-// putJSON marshals v and writes it at key.
-func putJSON(ctx context.Context, kv jetstream.KeyValue, key string, v any) error {
+// PutJSON marshals v and writes it at key.
+func PutJSON(ctx context.Context, kv jetstream.KeyValue, key string, v any) error {
 	data, err := json.Marshal(v)
 	if err != nil {
 		return err
@@ -190,8 +190,8 @@ func putJSON(ctx context.Context, kv jetstream.KeyValue, key string, v any) erro
 	return nil
 }
 
-// keysWithPrefix returns all KV keys under prefix, sorted.
-func keysWithPrefix(ctx context.Context, kv jetstream.KeyValue, prefix string) ([]string, error) {
+// KeysWithPrefix returns all KV keys under prefix, sorted.
+func KeysWithPrefix(ctx context.Context, kv jetstream.KeyValue, prefix string) ([]string, error) {
 	keys, err := kv.Keys(ctx)
 	if errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, nil

spinifex/domains/ecs/tasks.go

@@ -34,7 +34,7 @@ func (s *Service) RunTask(ctx context.Context, input *ecs.RunTaskInput, accountI
 	}
 
 	var clusterRec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
 	if err != nil {
 		return nil, err
 	}
@@ -82,7 +82,7 @@ func (s *Service) RunTask(ctx context.Context, input *ecs.RunTaskInput, accountI
 			}
 		}
 
-		if err := putJSON(ctx, kv, TaskKey(cluster, taskID), rec); err != nil {
+		if err := taskdefinition.PutJSON(ctx, kv, TaskKey(cluster, taskID), rec); err != nil {
 			return nil, err
 		}
 		if err := s.publishAssign(ctx, kv, accountID, cluster, inst.InstanceID, rec, taskDef); err != nil {
@@ -166,7 +166,7 @@ func (s *Service) provisionTaskENI(ctx context.Context, kv jetstream.KeyValue, a
 			rec.DesiredStatus = TaskStatusStopped
 			rec.StoppedReason = fmt.Sprintf("ENI attach failed: %s", err)
 			rec.StoppedAt = time.Now().UTC()
-			if perr := putJSON(ctx, kv, TaskKey(cluster, rec.TaskID), rec); perr != nil {
+			if perr := taskdefinition.PutJSON(ctx, kv, TaskKey(cluster, rec.TaskID), rec); perr != nil {
 				slog.ErrorContext(ctx, "ECS RunTask: persist ENI rollback failed", "task", rec.TaskID, "err", perr)
 			}
 		}
@@ -271,7 +271,7 @@ func (s *Service) publishAssign(ctx context.Context, kv jetstream.KeyValue, acco
 	for _, c := range td.Containers {
 		msg.Containers = append(msg.Containers, containerToAssign(c))
 	}
-	return putJSON(ctx, kv, AssignmentKey(cluster, instanceID, rec.TaskID), &msg)
+	return taskdefinition.PutJSON(ctx, kv, AssignmentKey(cluster, instanceID, rec.TaskID), &msg)
 }
 
 // DescribeTasks returns task records for the named tasks in a cluster.
@@ -285,7 +285,7 @@ func (s *Service) DescribeTasks(ctx context.Context, input *ecs.DescribeTasksInp
 	for _, ref := range awsStringSlice(input.Tasks) {
 		taskID := ContainerInstanceShortID(ref)
 		var rec TaskRecord
-		found, err := getJSON(ctx, kv, TaskKey(cluster, taskID), &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, TaskKey(cluster, taskID), &rec)
 		if err != nil {
 			return nil, err
 		}
@@ -308,14 +308,14 @@ func (s *Service) ListTasks(ctx context.Context, input *ecs.ListTasksInput, acco
 		return nil, err
 	}
 	var clusterRec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
 	if err != nil {
 		return nil, err
 	}
 	if !found {
 		return nil, errors.New(awserrors.ErrorECSClusterNotFound)
 	}
-	keys, err := keysWithPrefix(ctx, kv, TasksPrefix(cluster))
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, TasksPrefix(cluster))
 	if err != nil {
 		return nil, err
 	}
@@ -323,7 +323,7 @@ func (s *Service) ListTasks(ctx context.Context, input *ecs.ListTasksInput, acco
 	arns := make([]string, 0, len(keys))
 	for _, k := range keys {
 		var rec TaskRecord
-		found, err := getJSON(ctx, kv, k, &rec)
+		found, err := taskdefinition.GetJSON(ctx, kv, k, &rec)
 		if err != nil {
 			return nil, err
 		}

spinifex/domains/ecs/tasks_gc.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"log/slog"
 	"strings"
 	"time"
@@ -52,7 +53,7 @@ func (sc *Scheduler) sweepStoppedTasks(ctx context.Context) (time.Duration, erro
 // the ENI release for any that still owe one. A missing StoppedAt or an owed
 // (unreleased) ENI both block deletion; the latter until the release succeeds.
 func (s *Service) sweepStoppedBucket(ctx context.Context, kv jetstream.KeyValue, accountID string, now time.Time, retention time.Duration) (int, time.Duration, error) {
-	keys, err := keysWithPrefix(ctx, kv, "clusters/")
+	keys, err := taskdefinition.KeysWithPrefix(ctx, kv, "clusters/")
 	if err != nil {
 		return 0, 0, err
 	}
@@ -64,7 +65,7 @@ func (s *Service) sweepStoppedBucket(ctx context.Context, kv jetstream.KeyValue,
 			continue
 		}
 		var task TaskRecord
-		found, gerr := getJSON(ctx, kv, k, &task)
+		found, gerr := taskdefinition.GetJSON(ctx, kv, k, &task)
 		if gerr != nil || !found {
 			continue
 		}
@@ -116,7 +117,7 @@ func (s *Service) retryTaskENIRelease(ctx context.Context, kv jetstream.KeyValue
 		task.ENIReleaseAttempts++
 		task.ENIReleaseNextTry = now.Add(eniReleaseBackoff(task.ENIReleaseAttempts))
 	}
-	if perr := putJSON(ctx, kv, key, task); perr != nil {
+	if perr := taskdefinition.PutJSON(ctx, kv, key, task); perr != nil {
 		slog.Warn("ECS sweep: persist ENI retry state failed", "key", key, "err", perr)
 	}
 }

spinifex/domains/ecs/tasks_gc_test.go

@@ -3,6 +3,7 @@ package ecs
 import (
 	"errors"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 	"time"
 
@@ -19,7 +20,7 @@ func TestSweepStoppedBucket_PrunesOnlyStale(t *testing.T) {
 			TaskID: id, Cluster: "web", ARN: TaskARN(testRegion, testAccountID, "web", id),
 			LastStatus: status, DesiredStatus: status, StoppedAt: stoppedAt,
 		}
-		require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", id), &rec))
+		require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", id), &rec))
 	}
 
 	put("stale", TaskStatusStopped, now.Add(-2*time.Hour))   // older than retention -> prune
@@ -36,7 +37,7 @@ func TestSweepStoppedBucket_PrunesOnlyStale(t *testing.T) {
 
 	exists := func(id string) bool {
 		var rec TaskRecord
-		found, gerr := getJSON(t.Context(), kv, TaskKey("web", id), &rec)
+		found, gerr := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", id), &rec)
 		require.NoError(t, gerr)
 		return found
 	}
@@ -58,7 +59,7 @@ func TestSweepStoppedBucket_RetriesOwedENI_Succeeds(t *testing.T) {
 		NetworkMode: NetworkModeAwsvpc, ENIID: "eni-1", ENIAttachmentID: "att-1",
 		StoppedAt: now.Add(-2 * time.Hour), // well past retention
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec))
 
 	pruned, due, err := svc.sweepStoppedBucket(t.Context(), kv, testAccountID, now, time.Hour)
 	require.NoError(t, err)
@@ -67,7 +68,7 @@ func TestSweepStoppedBucket_RetriesOwedENI_Succeeds(t *testing.T) {
 	assert.Equal(t, 1, eni.releaseCalls)
 
 	var got TaskRecord
-	found, gerr := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &got)
+	found, gerr := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &got)
 	require.NoError(t, gerr)
 	require.True(t, found)
 	assert.Equal(t, "eni-1", got.ENIID, "the identity survives release as the forensic record")
@@ -88,7 +89,7 @@ func TestSweepStoppedBucket_RetryFails_BacksOff(t *testing.T) {
 		NetworkMode: NetworkModeAwsvpc, ENIID: "eni-1",
 		StoppedAt: now.Add(-2 * time.Hour),
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec))
 
 	pruned, due, err := svc.sweepStoppedBucket(t.Context(), kv, testAccountID, now, time.Hour)
 	require.NoError(t, err)
@@ -96,7 +97,7 @@ func TestSweepStoppedBucket_RetryFails_BacksOff(t *testing.T) {
 	assert.Equal(t, eniReleaseBackoffBase, due)
 
 	var got TaskRecord
-	found, gerr := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &got)
+	found, gerr := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &got)
 	require.NoError(t, gerr)
 	require.True(t, found)
 	assert.Equal(t, "eni-1", got.ENIID, "a failed retry is still owed")
@@ -118,7 +119,7 @@ func TestSweepStoppedBucket_RetryNotDueYet_Skipped(t *testing.T) {
 		StoppedAt:         now.Add(-2 * time.Hour),
 		ENIReleaseNextTry: now.Add(10 * time.Minute),
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec))
 
 	pruned, due, err := svc.sweepStoppedBucket(t.Context(), kv, testAccountID, now, time.Hour)
 	require.NoError(t, err)
@@ -142,7 +143,7 @@ func TestSweepStoppedBucket_HonoursPerPassCap(t *testing.T) {
 			NetworkMode: NetworkModeAwsvpc, ENIID: "eni-" + id,
 			StoppedAt: now.Add(-2 * time.Hour),
 		}
-		require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", id), &rec))
+		require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", id), &rec))
 	}
 
 	_, due, err := svc.sweepStoppedBucket(t.Context(), kv, testAccountID, now, time.Hour)
@@ -173,7 +174,7 @@ func TestSweepStoppedBucket_CappedOverflow_AsksForSweepInterval(t *testing.T) {
 			NetworkMode: NetworkModeAwsvpc, ENIID: "eni-" + id,
 			StoppedAt: now.Add(-2 * time.Hour),
 		}
-		require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", id), &rec))
+		require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", id), &rec))
 	}
 
 	_, due, err := svc.sweepStoppedBucket(t.Context(), kv, testAccountID, now, time.Hour)
@@ -196,7 +197,7 @@ func TestSweepStoppedBucket_PrunesReleased_KeepsOwed(t *testing.T) {
 		NetworkMode: NetworkModeAwsvpc, ENIID: "eni-released", ENIReleased: true,
 		StoppedAt: now.Add(-2 * time.Hour), // well past retention
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "released"), &released))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "released"), &released))
 
 	owed := TaskRecord{
 		TaskID: "owed", Cluster: "web", ARN: TaskARN(testRegion, testAccountID, "web", "owed"),
@@ -204,7 +205,7 @@ func TestSweepStoppedBucket_PrunesReleased_KeepsOwed(t *testing.T) {
 		NetworkMode: NetworkModeAwsvpc, ENIID: "eni-owed", ENIReleased: false,
 		StoppedAt: now.Add(-2 * time.Hour), // also well past retention, but still owed
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "owed"), &owed))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "owed"), &owed))
 
 	pruned, _, err := svc.sweepStoppedBucket(t.Context(), kv, testAccountID, now, time.Hour)
 	require.NoError(t, err)
@@ -212,7 +213,7 @@ func TestSweepStoppedBucket_PrunesReleased_KeepsOwed(t *testing.T) {
 
 	exists := func(id string) bool {
 		var rec TaskRecord
-		found, gerr := getJSON(t.Context(), kv, TaskKey("web", id), &rec)
+		found, gerr := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", id), &rec)
 		require.NoError(t, gerr)
 		return found
 	}

spinifex/domains/eks/awsapi/registration.go

@@ -3,11 +3,9 @@ package awsapi
 import (
 	"encoding/json"
 	"errors"
-	"io"
 	"log/slog"
 	"net/http"
 
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/passrole"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -64,17 +62,6 @@ func CallerFromActor(actor dispatch.Actor) Caller {
 	}
 }
 
-func readBoundedBody(r *http.Request) ([]byte, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return body, nil
-}
-
 // dispatch runs the action and renames the generic parameter error to the code
 // EKS declares. The internal routes keep theirs: the CP-VM agents match on it.
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
@@ -139,7 +126,7 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		return errors.New(awserrors.ErrorInternalError)
 	}
 
-	body, err := readBoundedBody(r)
+	body, err := dispatch.ReadBoundedBody(r)
 	if err != nil {
 		slog.ErrorContext(r.Context(), "EKS dispatch: failed to read body", "err", err)
 		return err

spinifex/domains/eks/cluster/delete_cluster_review_test.go

@@ -205,6 +205,8 @@ func TestS2_EKSLastNodegroupDeletedResumesTheClusterDelete(t *testing.T) {
 		ClusterName: aws.String("alpha"), NodegroupName: aws.String("ng1"),
 	}, testAccountID)
 	require.NoError(t, err)
+	_, err = f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
+	require.NoError(t, err)
 	f.svc.WaitTeardowns()
 
 	_, err = GetClusterMeta(t.Context(), f.kv, "alpha")

spinifex/domains/eks/cluster/delete_cluster_test.go

@@ -105,6 +105,9 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture {
 	require.NoError(t, err)
 	// Tight Ready-gating knobs so nodegroup launches resolve in tests without the
 	// production 10m timeout; tests drive meta.NodeCount to simulate the reconciler.
+	svc.noDeleteKick = true
+	svc.workersEndWait = 20 * time.Millisecond
+	svc.workersEndPoll = time.Millisecond
 	svc.nodegroupReadyTimeout = 50 * time.Millisecond
 	svc.nodegroupReadyPoll = 2 * time.Millisecond
 	// Tight worker-launch-retry knobs so a launch-failure retry test resolves

spinifex/domains/eks/cluster/eks_nodegroup_deleting_reaper.go

@@ -0,0 +1,284 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// A DELETING nodegroup becomes DELETE_FAILED once it has been deleting this
+// long and this many passes have failed. Vars for tests.
+var (
+	nodegroupDeleteFailAfter    = 30 * time.Minute
+	nodegroupDeleteFailAttempts = 5
+)
+
+// A pass waits this long, polling, for ended workers to be proven gone.
+const (
+	defaultWorkersEndWait = 90 * time.Second
+	defaultWorkersEndPoll = 3 * time.Second
+)
+
+// errWorkersEnding means workers were told to end but are not yet proven gone.
+var errWorkersEnding = errors.New("workers still ending")
+
+// EKSNodegroupDeletingReaper finishes recorded nodegroup deletes: it ends the
+// workers, removes the nodegroup's own instance profile and only then the
+// record, so a nodegroup is never gone while something it made still is.
+type EKSNodegroupDeletingReaper struct {
+	svc *EKSServiceImpl
+}
+
+var _ vm.Reaper = (*EKSNodegroupDeletingReaper)(nil)
+
+// NewNodegroupDeletingReaper builds the DELETING nodegroup converger.
+func (s *EKSServiceImpl) NewNodegroupDeletingReaper() *EKSNodegroupDeletingReaper {
+	return &EKSNodegroupDeletingReaper{svc: s}
+}
+
+func (r *EKSNodegroupDeletingReaper) Class() string         { return "eks-nodegroup-deleting" }
+func (r *EKSNodegroupDeletingReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
+
+// Sweep hands every due DELETING nodegroup to a background converger and
+// returns how many it started. The lease decides which node runs each one.
+func (r *EKSNodegroupDeletingReaper) Sweep(ctx context.Context) (int, error) {
+	if !r.svc.depsReadyForOrchestration() {
+		return 0, nil
+	}
+	js, err := jetstream.New(r.svc.deps.NATSConn)
+	if err != nil {
+		return 0, fmt.Errorf("jetstream: %w", err)
+	}
+	names, err := accountBucketNames(ctx, r.svc.deps.NATSConn)
+	if err != nil {
+		return 0, fmt.Errorf("enumerate account buckets: %w", err)
+	}
+
+	started := 0
+	for _, name := range names {
+		select {
+		case <-ctx.Done():
+			return started, ctx.Err()
+		default:
+		}
+		accountID := strings.TrimPrefix(name, KVBucketEKSAccountPrefix)
+		acctKV, err := js.KeyValue(ctx, name)
+		if err != nil {
+			continue
+		}
+		clusters, err := listClusterNames(ctx, acctKV)
+		if err != nil {
+			continue
+		}
+		for _, cluster := range clusters {
+			recs, err := ListNodegroupRecords(ctx, acctKV, cluster)
+			if err != nil {
+				continue
+			}
+			for _, rec := range recs {
+				if r.due(rec) && r.svc.startNodegroupDelete(accountID, acctKV, cluster, rec.Name) {
+					started++
+				}
+			}
+		}
+	}
+	return started, nil
+}
+
+// due reports whether rec is DELETING and past its retry backoff.
+func (r *EKSNodegroupDeletingReaper) due(rec *NodegroupRecord) bool {
+	if rec.Status != eks.NodegroupStatusDeleting {
+		return false
+	}
+	if rec.DeleteReapAttempts == 0 || rec.LastDeleteReapAttempt.IsZero() {
+		return true
+	}
+	return time.Since(rec.LastDeleteReapAttempt) >= deleteReapBackoff(deletingReapMinAge, rec.DeleteReapAttempts)
+}
+
+// startNodegroupDelete runs one converger pass in the background and reports
+// whether it started; a pass already running here or on another node refuses.
+func (s *EKSServiceImpl) startNodegroupDelete(accountID string, kv jetstream.KeyValue, cluster, ng string) bool {
+	release, _, ok := s.holdTeardown(accountID, cluster+"/nodegroup/"+ng, false)
+	if !ok {
+		return false
+	}
+	s.teardownWG.Go(func() {
+		defer release()
+		ctx, cancel := context.WithTimeout(s.bgCtx, teardownPassTimeout)
+		defer cancel()
+		if err := s.convergeNodegroupDeletion(ctx, accountID, kv, cluster, ng); err != nil {
+			slog.Warn("eks: nodegroup delete incomplete, the record stays DELETING for retry",
+				"cluster", cluster, "nodegroup", ng, "account", accountID, "err", err)
+		}
+	})
+	return true
+}
+
+// WaitNodegroupDeletes blocks until every converger this process started has
+// returned. Intended for tests.
+func (s *EKSServiceImpl) WaitNodegroupDeletes() { s.teardownWG.Wait() }
+
+// mutateNodegroup applies fn to the stored record under compare-and-swap. fn
+// returns false to leave the record unchanged.
+func (s *EKSServiceImpl) mutateNodegroup(ctx context.Context, kv jetstream.KeyValue, cluster, ng string, fn func(*NodegroupRecord) bool) (*NodegroupRecord, error) {
+	for range ngCASMaxRetries {
+		rec, rev, err := getNodegroupEntry(ctx, kv, cluster, ng)
+		if err != nil {
+			return nil, err
+		}
+		if !fn(rec) {
+			return rec, nil
+		}
+		ok, err := s.casPutNodegroup(ctx, kv, rec, rev)
+		if err != nil {
+			return nil, err
+		}
+		if ok {
+			return rec, nil
+		}
+	}
+	return nil, fmt.Errorf("nodegroup %s/%s: concurrent writers kept winning", cluster, ng)
+}
+
+// convergeNodegroupDeletion finishes one recorded delete. It acts only on a
+// record that is DELETING when re-read, and keeps the record on any failure.
+func (s *EKSServiceImpl) convergeNodegroupDeletion(ctx context.Context, accountID string, kv jetstream.KeyValue, cluster, ng string) error {
+	rec, err := s.mutateNodegroup(ctx, kv, cluster, ng, func(r *NodegroupRecord) bool {
+		if r.Status != eks.NodegroupStatusDeleting {
+			return false
+		}
+		r.DeleteReapAttempts++
+		r.LastDeleteReapAttempt = time.Now().UTC()
+		return true
+	})
+	if err != nil {
+		if errors.Is(err, ErrNodegroupNotFound) {
+			return nil
+		}
+		return err
+	}
+	if rec.Status != eks.NodegroupStatusDeleting {
+		return nil
+	}
+
+	if err := s.releaseNodegroupResources(ctx, accountID, kv, rec); err != nil {
+		if errors.Is(err, errWorkersEnding) {
+			s.noteWorkersEnding(ctx, kv, cluster, ng, err)
+			return err
+		}
+		s.noteNodegroupDeleteFailure(ctx, kv, rec, err)
+		return err
+	}
+	if err := DeleteNodegroupRecord(ctx, kv, cluster, ng); err != nil {
+		return err
+	}
+	// A cluster delete waits on its node groups; the last one going lets it run.
+	if meta, err := GetClusterMeta(ctx, kv, cluster); err == nil && meta.Status == ClusterStatusDeleting {
+		if rest, err := ListNodegroupRecords(ctx, kv, cluster); err == nil && len(rest) == 0 {
+			s.startTeardown(accountID, cluster)
+		}
+	}
+	return nil
+}
+
+// releaseNodegroupResources ends the workers, then removes the profile. The
+// profile and the recorded worker ids go only once every worker is proven
+// terminated or absent, since an accepted terminate is only a request.
+func (s *EKSServiceImpl) releaseNodegroupResources(ctx context.Context, accountID string, kv jetstream.KeyValue, rec *NodegroupRecord) error {
+	if len(rec.InstanceIDs) > 0 {
+		if err := s.deps.Worker.TerminateWorkerInstances(ctx, rec.InstanceIDs, accountID); err != nil {
+			return fmt.Errorf("terminate workers: %w", err)
+		}
+		if err := s.awaitWorkersGone(ctx, rec.InstanceIDs, accountID); err != nil {
+			return err
+		}
+		if _, err := s.mutateNodegroup(ctx, kv, rec.ClusterName, rec.Name, func(r *NodegroupRecord) bool {
+			r.InstanceIDs = nil
+			return true
+		}); err != nil {
+			return fmt.Errorf("record workers ended: %w", err)
+		}
+	}
+	return s.deleteNodegroupProfile(ctx, accountID, kv, rec)
+}
+
+// awaitWorkersGone polls until no worker is live, up to the end wait,
+// and returns errWorkersEnding if some still are.
+func (s *EKSServiceImpl) awaitWorkersGone(ctx context.Context, ids []string, accountID string) error {
+	wait, poll := s.workersEndWait, s.workersEndPoll
+	if wait <= 0 {
+		wait = defaultWorkersEndWait
+	}
+	if poll <= 0 {
+		poll = defaultWorkersEndPoll
+	}
+	deadline := time.Now().Add(wait)
+	for {
+		live, err := s.deps.Worker.LiveWorkerInstances(ctx, ids, accountID)
+		if err != nil {
+			return fmt.Errorf("confirm workers ended: %w", err)
+		}
+		if len(live) == 0 {
+			return nil
+		}
+		if !time.Now().Before(deadline) {
+			return fmt.Errorf("%w: %s", errWorkersEnding, strings.Join(live, ","))
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(poll):
+		}
+	}
+}
+
+// noteWorkersEnding handles workers that are not yet gone. Early on that is no
+// failed attempt: the next sweep checks again without backoff. Once the group
+// has been deleting past nodegroupDeleteFailAfter it settles DELETE_FAILED
+// naming the instances, keeping the profile and record for a customer retry.
+func (s *EKSServiceImpl) noteWorkersEnding(ctx context.Context, kv jetstream.KeyValue, cluster, ng string, cause error) {
+	_, err := s.mutateNodegroup(ctx, kv, cluster, ng, func(r *NodegroupRecord) bool {
+		if r.Status != eks.NodegroupStatusDeleting {
+			return false
+		}
+		if time.Since(r.DeletingSince) >= nodegroupDeleteFailAfter {
+			r.Status = eks.NodegroupStatusDeleteFailed
+			r.StatusReason = "delete: " + cause.Error()
+			r.ModifiedAt = time.Now().UTC()
+			return true
+		}
+		r.DeleteReapAttempts--
+		r.LastDeleteReapAttempt = time.Time{}
+		return true
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "eks: record workers still ending", "cluster", cluster, "nodegroup", ng, "err", err)
+	}
+}
+
+// noteNodegroupDeleteFailure records why a pass failed and, once the group has
+// been deleting long enough and failed often enough, settles it DELETE_FAILED.
+func (s *EKSServiceImpl) noteNodegroupDeleteFailure(ctx context.Context, kv jetstream.KeyValue, rec *NodegroupRecord, cause error) {
+	_, err := s.mutateNodegroup(ctx, kv, rec.ClusterName, rec.Name, func(r *NodegroupRecord) bool {
+		if r.Status != eks.NodegroupStatusDeleting {
+			return false
+		}
+		r.StatusReason = "delete: " + cause.Error()
+		r.ModifiedAt = time.Now().UTC()
+		if r.DeleteReapAttempts >= nodegroupDeleteFailAttempts && time.Since(r.DeletingSince) >= nodegroupDeleteFailAfter {
+			r.Status = eks.NodegroupStatusDeleteFailed
+		}
+		return true
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "eks: record nodegroup delete failure", "cluster", rec.ClusterName, "nodegroup", rec.Name, "err", err)
+	}
+}

spinifex/domains/eks/cluster/nodegroup.go

@@ -379,6 +379,12 @@ func (s *EKSServiceImpl) createNodegroup(ctx context.Context, acctKV jetstream.K
 		ModifiedAt:     now,
 	}
 
+	// Recorded before the claim, so a name is durable before any worker launches.
+	if rec.NodeRole != "" {
+		rec.InstanceProfileID = uuid.NewV4().String()
+		rec.InstanceProfileName = nodegroupProfilePrefix + rec.InstanceProfileID
+	}
+
 	// Atomically claim the record; concurrent/duplicate CreateNodegroup requests lose here.
 	owned, err := ClaimNodegroupRecord(ctx, acctKV, rec)
 	if err != nil {
@@ -716,7 +722,7 @@ func (s *EKSServiceImpl) launchOneWorker(ctx context.Context, rec *NodegroupReco
 	// Attach the node role's instance profile so IMDS serves the role to the ECR
 	// credential provider; without it worker pulls from the internal ECR get 401.
 	if s.iamEnsurer() != nil && rec.NodeRole != "" {
-		profileARN, perr := s.ensureNodeInstanceProfile(accountID, rec.NodeRole)
+		profileARN, perr := s.nodeProfileForLaunch(accountID, rec)
 		if perr != nil {
 			return "", fmt.Errorf("ensure node instance profile: %w", perr)
 		}
@@ -855,14 +861,10 @@ func gatewayHostIP(gatewayURL string) string {
 	return host
 }
 
-// ensureNodeInstanceProfile guarantees an IAM instance profile named for the
-// nodegroup's node role exists and carries that role, returning its ARN. Workers
-// launch with this profile so IMDS serves the node role to the ECR credential
-// provider, mirroring the implicit instance profile real EKS creates for a node
-// role. Idempotent: concurrent worker launches converge on the same profile.
-func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string) (string, error) {
-	// Resolution compares the supplied ARN against the stored one, so an invented
-	// path cannot bind a profile to the role its trailing name happens to match.
+// resolveNodeRoleName returns the name of the stored role nodeRoleARN names.
+// Resolution compares the supplied ARN against the stored one, so an invented
+// path cannot bind a profile to the role its trailing name happens to match.
+func (s *EKSServiceImpl) resolveNodeRoleName(accountID, nodeRoleARN string) (string, error) {
 	_, roleName, err := auth.ResolveRoleARN(nodeRoleARN, func(account, name string) (string, error) {
 		// The profile is created under the caller's account, so a role from
 		// another account would silently bind a name that account does not own.
@@ -881,6 +883,19 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 	if err != nil {
 		return "", fmt.Errorf("node role ARN %q: %w", nodeRoleARN, err)
 	}
+	return roleName, nil
+}
+
+// ensureNodeInstanceProfile guarantees an IAM instance profile named for the
+// nodegroup's node role exists and carries that role, returning its ARN. Workers
+// launch with this profile so IMDS serves the node role to the ECR credential
+// provider, mirroring the implicit instance profile real EKS creates for a node
+// role. Idempotent: concurrent worker launches converge on the same profile.
+func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string) (string, error) {
+	roleName, err := s.resolveNodeRoleName(accountID, nodeRoleARN)
+	if err != nil {
+		return "", err
+	}
 	profileName := roleName
 
 	out, err := s.iamEnsurer().GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{
@@ -1018,6 +1033,9 @@ func (s *EKSServiceImpl) reconcileNodegroup(ctx context.Context, acctKV jetstrea
 			}
 			return nil, err
 		}
+		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusDeleteFailed {
+			return nil, errors.New(awserrors.ErrorEKSResourceInUse)
+		}
 		applyScalingUpdate(rec, scaling)
 		if labels != nil {
 			rec.Labels = applyLabelUpdate(rec.Labels, labels)
@@ -1227,33 +1245,49 @@ func (s *EKSServiceImpl) deleteNodegroup(ctx context.Context, acctKV jetstream.K
 	if err := requireCluster(ctx, acctKV, cluster); err != nil {
 		return nil, err
 	}
-	rec, err := GetNodegroupRecord(ctx, acctKV, cluster, ng)
+	rec, err := s.markNodegroupDeleting(ctx, acctKV, cluster, ng)
 	if err != nil {
-		if errors.Is(err, ErrNodegroupNotFound) {
-			return nil, nodegroupNotFound(ng)
-		}
 		return nil, err
 	}
-
-	if len(rec.InstanceIDs) > 0 {
-		if err := s.deps.Worker.TerminateWorkerInstances(ctx, rec.InstanceIDs, accountID); err != nil {
-			return nil, fmt.Errorf("terminate workers: %w", err)
-		}
+	if !s.noDeleteKick {
+		s.startNodegroupDelete(accountID, acctKV, cluster, ng)
 	}
-	if err := DeleteNodegroupRecord(ctx, acctKV, cluster, ng); err != nil {
-		return nil, err
-	}
-	// A cluster delete waits on its node groups; the last one going lets it run.
-	if meta, err := GetClusterMeta(ctx, acctKV, cluster); err == nil && meta.Status == ClusterStatusDeleting {
-		if rest, err := ListNodegroupRecords(ctx, acctKV, cluster); err == nil && len(rest) == 0 {
-			s.startTeardown(accountID, cluster)
-		}
-	}
-
-	rec.Status = eks.NodegroupStatusDeleting
 	return &eks.DeleteNodegroupOutput{Nodegroup: nodegroupRecordToAWS(rec)}, nil
 }
 
+// markNodegroupDeleting records DELETING and returns. The workers, the profile
+// and the record itself are the nodegroup reaper's to remove, so the request
+// never waits on work that scales with the group (a launching group is refused).
+func (s *EKSServiceImpl) markNodegroupDeleting(ctx context.Context, kv jetstream.KeyValue, cluster, ng string) (*NodegroupRecord, error) {
+	for range ngCASMaxRetries {
+		rec, rev, err := getNodegroupEntry(ctx, kv, cluster, ng)
+		if err != nil {
+			if errors.Is(err, ErrNodegroupNotFound) {
+				return nil, nodegroupNotFound(ng)
+			}
+			return nil, err
+		}
+		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusCreating {
+			return nil, errors.New(awserrors.ErrorEKSResourceInUse)
+		}
+		now := time.Now().UTC()
+		rec.Status = eks.NodegroupStatusDeleting
+		rec.StatusReason = ""
+		rec.DeletingSince = now
+		rec.DeleteReapAttempts = 0
+		rec.LastDeleteReapAttempt = time.Time{}
+		rec.ModifiedAt = now
+		ok, err := s.casPutNodegroup(ctx, kv, rec, rev)
+		if err != nil {
+			return nil, err
+		}
+		if ok {
+			return rec, nil
+		}
+	}
+	return nil, errors.New(awserrors.ErrorServerInternal)
+}
+
 // decryptNodeToken reads + decrypts the cluster's K3s join token from KV.
 func (s *EKSServiceImpl) decryptNodeToken(ctx context.Context, kv jetstream.KeyValue, cluster string) (string, error) {
 	if kv == nil {
@@ -1496,8 +1530,12 @@ func nodegroupRecordToAWS(rec *NodegroupRecord) *eks.Nodegroup {
 	// wrong; a nil slice here would marshal the key away entirely.
 	issues := make([]*eks.Issue, 0, 1)
 	if rec.StatusReason != "" {
+		code := eks.NodegroupIssueCodeNodeCreationFailure
+		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusDeleteFailed {
+			code = eks.NodegroupIssueCodeNodeTerminationFailure
+		}
 		issues = append(issues, &eks.Issue{
-			Code:        aws.String(eks.NodegroupIssueCodeNodeCreationFailure),
+			Code:        aws.String(code),
 			Message:     aws.String(rec.StatusReason),
 			ResourceIds: aws.StringSlice(rec.InstanceIDs),
 		})

spinifex/domains/eks/cluster/nodegroup_delete_failure_test.go

@@ -0,0 +1,255 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func (f *fakeWorkerLauncher) terminateCallCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return len(f.terminateCalls)
+}
+
+func requireNodegroupStatus(t *testing.T, f *eksServiceFixture, cluster, ng, want string) *NodegroupRecord {
+	t.Helper()
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, cluster, ng)
+	require.NoError(t, err)
+	require.Equal(t, want, rec.Status)
+	return rec
+}
+
+func TestNodegroupDeletingReaper_SweepWithoutOrchestrationDepsStartsNothing(t *testing.T) {
+	t.Parallel()
+	n, err := (&EKSServiceImpl{}).NewNodegroupDeletingReaper().Sweep(t.Context())
+	require.NoError(t, err)
+	assert.Zero(t, n)
+}
+
+func TestNodegroupDeletingReaper_CancelledSweepLeavesTheDeleteRecorded(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	n, err := f.svc.NewNodegroupDeletingReaper().Sweep(ctx)
+	f.svc.WaitNodegroupDeletes()
+
+	require.ErrorIs(t, err, context.Canceled)
+	assert.Zero(t, n)
+	requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusDeleting)
+}
+
+func TestNodegroupDeletingReaper_OnlyDeletingGroupsPastBackoffAreStarted(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "active", 1)
+	createReadyNodegroup(t, f, "c1", "backoff", 1)
+	_, err := deleteNodegroupCall(f, "c1", "backoff")
+	require.NoError(t, err)
+	rec := requireNodegroupStatus(t, f, "c1", "backoff", eks.NodegroupStatusDeleting)
+	rec.DeleteReapAttempts = 3
+	rec.LastDeleteReapAttempt = time.Now().UTC()
+	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, rec))
+
+	assert.Zero(t, sweepNodegroups(t, f), "an ACTIVE group and a DELETING one inside its backoff are not due")
+	requireNodegroupStatus(t, f, "c1", "active", eks.NodegroupStatusActive)
+	assert.Zero(t, f.worker.terminateCallCount())
+}
+
+func TestConvergeNodegroupDeletion_MissingOrNotDeletingRecordIsLeftAlone(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+
+	require.NoError(t, f.svc.convergeNodegroupDeletion(t.Context(), testAccountID, f.kv, "c1", "never-existed"))
+	require.NoError(t, f.svc.convergeNodegroupDeletion(t.Context(), testAccountID, f.kv, "c1", "ng1"))
+
+	requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusActive)
+	assert.Zero(t, f.worker.terminateCallCount(), "a group nobody asked to delete keeps its workers")
+	assert.Contains(t, fixtureIAM(t, f).Profiles, mustProfileName(t, f, "c1", "ng1"))
+}
+
+func mustProfileName(t *testing.T, f *eksServiceFixture, cluster, ng string) string {
+	t.Helper()
+	rec, _ := ownedProfile(t, f, cluster, ng)
+	return rec.InstanceProfileName
+}
+
+func TestAwaitWorkersGone_CancelledWhileWorkersEndingReturnsTheContextError(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	f.worker.stillLive = []string{"i-slow"}
+	f.svc.workersEndWait = time.Hour
+	f.svc.workersEndPoll = time.Hour
+	ctx, cancel := context.WithCancel(t.Context())
+	time.AfterFunc(50*time.Millisecond, cancel)
+
+	err := f.svc.awaitWorkersGone(ctx, []string{"i-slow"}, testAccountID)
+
+	require.ErrorIs(t, err, context.Canceled)
+	assert.NotErrorIs(t, err, errWorkersEnding, "a cancelled wait is not a verdict on the workers")
+}
+
+func TestDeleteNodegroup_IAMReadFailureKeepsGroupDeletingAndProfileIntact(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	m.GetInstanceProfileErr = errors.New("iam read timed out")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	cur := requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusDeleting)
+	assert.Contains(t, cur.StatusReason, "get instance profile", "the failed read is recorded for the customer")
+	assert.Contains(t, m.Profiles, rec.InstanceProfileName, "an unreadable profile is not a missing one")
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+
+	m.GetInstanceProfileErr = nil
+	backdateNodegroupReap(t, f, "c1", "ng1")
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+}
+
+func TestDeleteNodegroup_ProfileAlreadyGoneStillFinishesTheGroup(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	delete(m.Profiles, rec.InstanceProfileName)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.Zero(t, m.RemoveRoleCalls)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+}
+
+func TestDeleteNodegroup_ProfileVanishingMidDeleteStillFinishesTheGroup(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	m := fixtureIAM(t, f)
+	m.DeleteInstanceProfileErr = awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile gone.")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.Equal(t, 1, m.RemoveRoleCalls)
+}
+
+func TestDeleteNodegroup_DeleteProfileFailureKeepsGroupDeletingUntilItClears(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	m.DeleteInstanceProfileErr = errors.New("iam unavailable")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+	cur := requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusDeleting)
+	assert.Contains(t, cur.StatusReason, "delete instance profile")
+	assert.Empty(t, cur.InstanceIDs, "workers proven gone are not terminated again")
+
+	m.DeleteInstanceProfileErr = nil
+	backdateNodegroupReap(t, f, "c1", "ng1")
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+}
+
+func TestEnsureNodegroupProfile_RecreatesAMissingRecordedProfileAsItsOwn(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	delete(m.Profiles, rec.InstanceProfileName)
+
+	arn, err := f.svc.ensureNodegroupProfile(testAccountID, rec)
+
+	require.NoError(t, err)
+	assert.Contains(t, arn, rec.InstanceProfileName)
+	got := m.Profiles[rec.InstanceProfileName]
+	require.NotNil(t, got)
+	assert.True(t, profileOwnedBy(got, rec), "a re-created profile carries this incarnation's tags")
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(got))
+}
+
+func TestEnsureNodegroupProfile_RefusesAProfileThatIsNotThisGroups(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, p := ownedProfile(t, f, "c1", "ng1")
+	p.Tags = nil
+	callsBefore := fixtureIAM(t, f).AddRoleToInstanceProfileCalls
+
+	_, err := f.svc.ensureNodegroupProfile(testAccountID, rec)
+
+	require.ErrorContains(t, err, "is not this nodegroup's")
+	assert.Equal(t, callsBefore, fixtureIAM(t, f).AddRoleToInstanceProfileCalls, "no role is attached to a profile EKS does not own")
+}
+
+func TestEnsureNodegroupProfile_IAMFailuresSurfaceWithoutHalfWork(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+
+	m.GetInstanceProfileErr = errors.New("iam read failed")
+	_, err := f.svc.ensureNodegroupProfile(testAccountID, rec)
+	require.ErrorContains(t, err, "get instance profile")
+	m.GetInstanceProfileErr = nil
+
+	delete(m.Profiles, rec.InstanceProfileName)
+	m.CreateInstanceProfileErr = errors.New("iam write failed")
+	_, err = f.svc.ensureNodegroupProfile(testAccountID, rec)
+	require.ErrorContains(t, err, "create instance profile")
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+}
+
+func TestMarkNodegroupDeleting_UnknownGroupIsResourceNotFound(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+
+	_, err := f.svc.markNodegroupDeleting(t.Context(), f.kv, "c1", "ghost")
+
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}

spinifex/domains/eks/cluster/nodegroup_delete_test.go

@@ -0,0 +1,662 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/aws/aws-sdk-go/service/iam"
+	iammock "github.com/mulgadc/spinifex/spinifex/domains/iam/mock"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	profileClusterTag   = "spinifex:eks-cluster"
+	profileNodegroupTag = "spinifex:eks-nodegroup"
+	fixtureNodeRoleName = "eks-node"
+)
+
+func fixtureIAM(t *testing.T, f *eksServiceFixture) *iammock.SystemInstanceRoleEnsurer {
+	t.Helper()
+	m, ok := f.svc.deps.IAM.(*iammock.SystemInstanceRoleEnsurer)
+	require.True(t, ok, "the fixture IAM must be the in-memory mock")
+	return m
+}
+
+// createReadyNodegroup creates a nodegroup whose n workers register Ready.
+func createReadyNodegroup(t *testing.T, f *eksServiceFixture, cluster, ng string, n int64) {
+	t.Helper()
+	markWorkersReady(t, f, cluster, ng, int(n))
+	_, err := f.svc.CreateNodegroup(context.Background(), createNGInput(cluster, ng, n), testAccountID)
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+}
+
+// createUnreadyNodegroup creates a nodegroup that never registers, so it ends
+// CREATE_FAILED holding the workers it launched.
+func createUnreadyNodegroup(t *testing.T, f *eksServiceFixture, cluster, ng string, n int64) {
+	t.Helper()
+	_, err := f.svc.CreateNodegroup(context.Background(), createNGInput(cluster, ng, n), testAccountID)
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+}
+
+func deleteNodegroupCall(f *eksServiceFixture, cluster, ng string) (*eks.DeleteNodegroupOutput, error) {
+	return f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
+		ClusterName: aws.String(cluster), NodegroupName: aws.String(ng),
+	}, testAccountID)
+}
+
+func describeNodegroupCall(f *eksServiceFixture, cluster, ng string) (*eks.DescribeNodegroupOutput, error) {
+	return f.svc.DescribeNodegroup(context.Background(), &eks.DescribeNodegroupInput{
+		ClusterName: aws.String(cluster), NodegroupName: aws.String(ng),
+	}, testAccountID)
+}
+
+func sweepNodegroups(t *testing.T, f *eksServiceFixture) int {
+	t.Helper()
+	n, err := f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
+	require.NoError(t, err)
+	f.svc.WaitNodegroupDeletes()
+	return n
+}
+
+// backdateNodegroupReap moves the last reap pass far enough back that any
+// retry backoff has elapsed.
+func backdateNodegroupReap(t *testing.T, f *eksServiceFixture, cluster, ng string) {
+	t.Helper()
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, cluster, ng)
+	require.NoError(t, err)
+	rec.LastDeleteReapAttempt = time.Now().UTC().Add(-24 * time.Hour)
+	rec.DeletingSince = time.Now().UTC().Add(-24 * time.Hour)
+	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, rec))
+}
+
+func requireErrorCode(t *testing.T, err error, want string) {
+	t.Helper()
+	require.Error(t, err)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok, "error %v carries no AWS code", err)
+	assert.Equal(t, want, code)
+}
+
+func profileTag(p *iam.InstanceProfile, key string) string {
+	for _, tg := range p.Tags {
+		if aws.StringValue(tg.Key) == key {
+			return aws.StringValue(tg.Value)
+		}
+	}
+	return ""
+}
+
+func profileHasTagValue(p *iam.InstanceProfile, value string) bool {
+	for _, tg := range p.Tags {
+		if aws.StringValue(tg.Value) == value {
+			return true
+		}
+	}
+	return false
+}
+
+func profileRoleNames(p *iam.InstanceProfile) []string {
+	var names []string
+	for _, r := range p.Roles {
+		names = append(names, aws.StringValue(r.RoleName))
+	}
+	return names
+}
+
+// ownedProfile returns the nodegroup's record and the instance profile it
+// names, failing the test if either is missing.
+func ownedProfile(t *testing.T, f *eksServiceFixture, cluster, ng string) (*NodegroupRecord, *iam.InstanceProfile) {
+	t.Helper()
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, cluster, ng)
+	require.NoError(t, err)
+	require.NotEmpty(t, rec.InstanceProfileName, "the record must name the profile it owns")
+	require.NotEmpty(t, rec.InstanceProfileID, "the record must carry the per-incarnation id")
+	p, ok := fixtureIAM(t, f).Profiles[rec.InstanceProfileName]
+	require.True(t, ok, "profile %q must exist", rec.InstanceProfileName)
+	return rec, p
+}
+
+func seedCustomerProfile(f *eksServiceFixture, t *testing.T, name, role string, tags map[string]string) {
+	t.Helper()
+	p := &iam.InstanceProfile{
+		InstanceProfileName: aws.String(name),
+		Arn:                 aws.String("arn:aws:iam::" + testAccountID + ":instance-profile/" + name),
+	}
+	if role != "" {
+		p.Roles = []*iam.Role{{RoleName: aws.String(role)}}
+	}
+	for k, v := range tags {
+		p.Tags = append(p.Tags, &iam.Tag{Key: aws.String(k), Value: aws.String(v)})
+	}
+	fixtureIAM(t, f).Profiles[name] = p
+}
+
+func TestDeleteNodegroup_ReturnsDeletingWithoutTerminatingInRequest(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+
+	out, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleting, aws.StringValue(out.Nodegroup.Status))
+
+	assert.Zero(t, f.worker.terminatedCount(), "the request must not wait for workers to end")
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err, "the record stays until the reaper has finished")
+	assert.Equal(t, eks.NodegroupStatusDeleting, rec.Status)
+
+	desc, err := describeNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleting, aws.StringValue(desc.Nodegroup.Status))
+}
+
+func TestDeleteNodegroup_ReaperEndsWorkersThenRemovesGroup(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, 1, sweepNodegroups(t, f))
+
+	require.Len(t, f.worker.terminateCalls, 1)
+	assert.ElementsMatch(t, rec.InstanceIDs, f.worker.terminateCalls[0])
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	_, ok := fixtureIAM(t, f).Profiles[rec.InstanceProfileName]
+	assert.False(t, ok, "the group is not gone until its profile is")
+}
+
+func TestDeleteNodegroup_RepeatedDeleteOnDeletingGroupIsResourceInUse(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	_, err = deleteNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
+}
+
+func TestDeleteNodegroup_DeletingGroupRefusesUpdates(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	_, err = f.svc.UpdateNodegroupConfig(context.Background(), &eks.UpdateNodegroupConfigInput{
+		ClusterName:   aws.String("c1"),
+		NodegroupName: aws.String("ng1"),
+		ScalingConfig: &eks.NodegroupScalingConfig{MinSize: aws.Int64(1), MaxSize: aws.Int64(3), DesiredSize: aws.Int64(3)},
+	}, testAccountID)
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
+	assert.Equal(t, 1, f.worker.runCount(), "a DELETING group must not launch workers")
+}
+
+func TestDeleteNodegroup_SameNameCreateWhileDeletingIsResourceInUse(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	_, err = f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
+}
+
+func TestDeleteNodegroup_DeleteClusterRefusedWhileGroupDeleting(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	_, err = f.svc.DeleteCluster(context.Background(), &eks.DeleteClusterInput{Name: aws.String("c1")}, testAccountID)
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
+}
+
+func TestDeleteNodegroup_BootReclaimLeavesDeletingGroupToReaper(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{
+		ClusterName: "c1", Name: "ng1", Status: eks.NodegroupStatusDeleting,
+		InstanceIDs: []string{"i-w1", "i-w2"}, NodeRole: fixtureNodeRoleARN,
+	}))
+
+	f.svc.reclaimOrphanedNodegroups(t.Context(), testAccountID, f.kv, "c1")
+
+	assert.Zero(t, f.worker.terminatedCount())
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleting, rec.Status)
+	assert.Equal(t, []string{"i-w1", "i-w2"}, rec.InstanceIDs)
+}
+
+func TestDeleteNodegroup_RepeatedReaperFailureEndsDeleteFailedWithHealthIssue(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	f.worker.termErr = errors.New("terminate refused")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	var rec *NodegroupRecord
+	for range 30 {
+		sweepNodegroups(t, f)
+		rec, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+		require.NoError(t, err, "a failing delete must leave the record, never drop it")
+		if rec.Status == eks.NodegroupStatusDeleteFailed {
+			break
+		}
+		backdateNodegroupReap(t, f, "c1", "ng1")
+	}
+	require.Equal(t, eks.NodegroupStatusDeleteFailed, rec.Status)
+
+	desc, err := describeNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	require.NotEmpty(t, desc.Nodegroup.Health.Issues, "DELETE_FAILED must say why")
+	assert.Equal(t, eks.NodegroupIssueCodeNodeTerminationFailure, aws.StringValue(desc.Nodegroup.Health.Issues[0].Code))
+
+	// The customer retries the delete once the cause is gone.
+	f.worker.termErr = nil
+	_, err = deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}
+
+func TestCreateNodegroup_OwnsATaggedProfileRecordedBeforeFirstLaunch(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	var seenAtLaunch string
+	f.worker.beforeRun = func(in *ec2.RunInstancesInput) {
+		if rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1"); err == nil && seenAtLaunch == "" {
+			seenAtLaunch = rec.InstanceProfileName
+		}
+	}
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+
+	rec, p := ownedProfile(t, f, "c1", "ng1")
+	assert.NotEqual(t, fixtureNodeRoleName, rec.InstanceProfileName, "the profile is the group's own, not the role-named one")
+	assert.Equal(t, rec.InstanceProfileName, seenAtLaunch, "the name must be durable before any worker launches")
+	assert.Equal(t, "c1", profileTag(p, profileClusterTag))
+	assert.Equal(t, "ng1", profileTag(p, profileNodegroupTag))
+	assert.True(t, profileHasTagValue(p, rec.InstanceProfileID), "a tag must carry the incarnation id")
+	for _, tg := range p.Tags {
+		assert.True(t, strings.HasPrefix(aws.StringValue(tg.Key), "spinifex:"), "tag %q: aws: keys are reserved, ownership tags must be spinifex:", aws.StringValue(tg.Key))
+	}
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(p))
+
+	require.Equal(t, 2, f.worker.runCount())
+	for _, in := range f.worker.runCalls {
+		require.NotNil(t, in.IamInstanceProfile)
+		assert.True(t, strings.HasSuffix(aws.StringValue(in.IamInstanceProfile.Arn), "/"+rec.InstanceProfileName),
+			"every worker launches with the group's own profile, got %s", aws.StringValue(in.IamInstanceProfile.Arn))
+	}
+}
+
+func TestCreateNodegroup_TwoGroupsOnOneRoleGetTwoProfiles(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createUnreadyNodegroup(t, f, "c1", "ng-a", 1)
+	createUnreadyNodegroup(t, f, "c1", "ng-b", 1)
+
+	recA, pA := ownedProfile(t, f, "c1", "ng-a")
+	recB, pB := ownedProfile(t, f, "c1", "ng-b")
+	assert.NotEqual(t, recA.InstanceProfileName, recB.InstanceProfileName)
+	assert.NotEqual(t, recA.InstanceProfileID, recB.InstanceProfileID)
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(pA))
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(pB))
+}
+
+func TestCreateNodegroup_RecreatedGroupOfSameNameGetsANewProfile(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createUnreadyNodegroup(t, f, "c1", "ng1", 1)
+	first, _ := ownedProfile(t, f, "c1", "ng1")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	createUnreadyNodegroup(t, f, "c1", "ng1", 1)
+	second, _ := ownedProfile(t, f, "c1", "ng1")
+	assert.NotEqual(t, first.InstanceProfileName, second.InstanceProfileName)
+	assert.NotEqual(t, first.InstanceProfileID, second.InstanceProfileID)
+	_, stillThere := fixtureIAM(t, f).Profiles[first.InstanceProfileName]
+	assert.False(t, stillThere, "the first incarnation's profile must have gone with it")
+}
+
+func TestDeleteNodegroup_AfterCreateFailedRemovesProfileAndGroup(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createUnreadyNodegroup(t, f, "c1", "ng1", 2)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	require.Equal(t, eks.NodegroupStatusCreateFailed, rec.Status)
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	_, ok := fixtureIAM(t, f).Profiles[rec.InstanceProfileName]
+	assert.False(t, ok)
+	assert.Equal(t, 2, f.worker.terminatedCount())
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}
+
+func TestDeleteNodegroup_ProfileSurvivesWhileWorkersAreLive(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	f.worker.termErr = errors.New("terminate refused")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	m := fixtureIAM(t, f)
+	assert.Contains(t, m.Profiles, rec.InstanceProfileName, "a profile in use by live workers must not be touched")
+	assert.Zero(t, m.RemoveRoleCalls)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+}
+
+func TestDeleteNodegroup_PartialProfileDeleteConvergesOnRetry(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	m.RemoveRoleFromInstanceProfileErr = errors.New("iam unavailable")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+	still, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err, "the group is not gone while its profile is")
+	assert.Equal(t, eks.NodegroupStatusDeleting, still.Status)
+
+	m.RemoveRoleFromInstanceProfileErr = nil
+	backdateNodegroupReap(t, f, "c1", "ng1")
+	sweepNodegroups(t, f)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}
+
+func TestDeleteNodegroup_CustomerRoleNamedProfileSurvives(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	seedCustomerProfile(f, t, fixtureNodeRoleName, fixtureNodeRoleName, nil)
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	m := fixtureIAM(t, f)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName, "the group's own profile goes")
+	require.Contains(t, m.Profiles, fixtureNodeRoleName, "a customer's profile is never EKS's to delete")
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[fixtureNodeRoleName]))
+	assert.Equal(t, 1, m.DeleteInstanceProfileCalls)
+}
+
+func TestDeleteNodegroup_RecordedProfileMissingOwnershipTagSurvives(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	_, err := m.UntagInstanceProfile(testAccountID, &iam.UntagInstanceProfileInput{
+		InstanceProfileName: aws.String(rec.InstanceProfileName),
+		TagKeys:             aws.StringSlice([]string{profileClusterTag}),
+	})
+	require.NoError(t, err)
+
+	_, err = deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	require.Contains(t, m.Profiles, rec.InstanceProfileName)
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[rec.InstanceProfileName]))
+	assert.Zero(t, m.RemoveRoleCalls)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}
+
+func TestDeleteNodegroup_OldLayoutGroupDeletesAndLeavesRoleNamedProfile(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	seedCustomerProfile(f, t, fixtureNodeRoleName, fixtureNodeRoleName, nil)
+	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{
+		ClusterName: "c1", Name: "old", Status: eks.NodegroupStatusActive,
+		InstanceIDs: []string{"i-old1"}, NodeRole: fixtureNodeRoleARN,
+	}))
+
+	_, err := deleteNodegroupCall(f, "c1", "old")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	require.Len(t, f.worker.terminateCalls, 1)
+	assert.Equal(t, []string{"i-old1"}, f.worker.terminateCalls[0])
+	_, err = describeNodegroupCall(f, "c1", "old")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	m := fixtureIAM(t, f)
+	require.Contains(t, m.Profiles, fixtureNodeRoleName)
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[fixtureNodeRoleName]))
+	assert.Zero(t, m.RemoveRoleCalls)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+}
+
+// No path deletes a profile unless it is recorded, tagged for this
+// incarnation, holds the record's node role and is named by no other record.
+func TestInvariant_NodegroupDeleteNeverRemovesAProfileItDoesNotOwn(t *testing.T) {
+	t.Parallel()
+	ownership := map[string]string{profileClusterTag: "c1", profileNodegroupTag: "ng1"}
+
+	t.Run("customer pre-creates a profile at a guessed name", func(t *testing.T) {
+		t.Parallel()
+		f := newEKSServiceFixture(t)
+		seedActiveClusterWithToken(t, f, "c1")
+		guesses := []string{
+			"eks-" + arn.EKSNodegroupDiscriminator(testAccountID, "c1", "ng1"),
+			"eks-c1-ng1",
+			"c1-ng1",
+			fixtureNodeRoleName,
+		}
+		for _, name := range guesses {
+			seedCustomerProfile(f, t, name, fixtureNodeRoleName, ownership)
+		}
+		createReadyNodegroup(t, f, "c1", "ng1", 1)
+		rec, _ := ownedProfile(t, f, "c1", "ng1")
+		require.NotContains(t, guesses, rec.InstanceProfileName, "the name must not be guessable")
+
+		_, err := deleteNodegroupCall(f, "c1", "ng1")
+		require.NoError(t, err)
+		sweepNodegroups(t, f)
+
+		m := fixtureIAM(t, f)
+		for _, name := range guesses {
+			require.Contains(t, m.Profiles, name, "customer profile %q was deleted", name)
+			assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[name]), "customer profile %q lost its role", name)
+		}
+	})
+
+	t.Run("recorded profile was swapped for one holding another role", func(t *testing.T) {
+		t.Parallel()
+		f := newEKSServiceFixture(t)
+		seedActiveClusterWithToken(t, f, "c1")
+		createReadyNodegroup(t, f, "c1", "ng1", 1)
+		rec, p := ownedProfile(t, f, "c1", "ng1")
+		p.Roles = []*iam.Role{{RoleName: aws.String("customer-role")}}
+
+		_, err := deleteNodegroupCall(f, "c1", "ng1")
+		require.NoError(t, err)
+		sweepNodegroups(t, f)
+
+		m := fixtureIAM(t, f)
+		require.Contains(t, m.Profiles, rec.InstanceProfileName)
+		assert.Equal(t, []string{"customer-role"}, profileRoleNames(m.Profiles[rec.InstanceProfileName]))
+		assert.Zero(t, m.RemoveRoleCalls)
+		assert.Zero(t, m.DeleteInstanceProfileCalls)
+	})
+
+	t.Run("another nodegroup record names the same profile", func(t *testing.T) {
+		t.Parallel()
+		f := newEKSServiceFixture(t)
+		seedActiveClusterWithToken(t, f, "c1")
+		createReadyNodegroup(t, f, "c1", "ng1", 1)
+		rec, _ := ownedProfile(t, f, "c1", "ng1")
+		require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{
+			ClusterName: "c1", Name: "ng2", Status: eks.NodegroupStatusActive, NodeRole: fixtureNodeRoleARN,
+			InstanceProfileName: rec.InstanceProfileName, InstanceProfileID: rec.InstanceProfileID,
+		}))
+
+		_, err := deleteNodegroupCall(f, "c1", "ng1")
+		require.NoError(t, err)
+		sweepNodegroups(t, f)
+
+		m := fixtureIAM(t, f)
+		require.Contains(t, m.Profiles, rec.InstanceProfileName)
+		assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[rec.InstanceProfileName]))
+		assert.Zero(t, m.DeleteInstanceProfileCalls)
+	})
+}
+
+func TestDeleteNodegroup_ProfileAndRecordKeptUntilWorkersProvenGone(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	f.worker.stillLive = []string{rec.InstanceIDs[0]}
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	m := fixtureIAM(t, f)
+	assert.Contains(t, m.Profiles, rec.InstanceProfileName)
+	assert.Zero(t, m.RemoveRoleCalls)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+	got, err := describeNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleting, aws.StringValue(got.Nodegroup.Status))
+	cur, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, rec.InstanceIDs, cur.InstanceIDs, "worker ids stay recorded until they are proven gone")
+	assert.Zero(t, cur.DeleteReapAttempts, "workers still ending is not a failed pass")
+
+	f.worker.mu.Lock()
+	f.worker.stillLive = nil
+	f.worker.mu.Unlock()
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+}
+
+func TestDeleteNodegroup_UndecidableWorkerStateKeepsProfile(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	f.worker.liveErr = errors.New("describe fan-out did not complete")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	m := fixtureIAM(t, f)
+	assert.Contains(t, m.Profiles, rec.InstanceProfileName)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+}
+
+func TestDeleteNodegroup_KickFinishesTheDeleteWithoutAReaperSweep(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	f.svc.noDeleteKick = false
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	f.svc.WaitNodegroupDeletes()
+
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
+}
+
+func TestDeleteNodegroup_WorkerStuckPastDeadlineEndsDeleteFailedKeepingProfile(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	stuck := rec.InstanceIDs[0]
+	f.worker.stillLive = []string{stuck}
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+	cur, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err)
+	require.Equal(t, eks.NodegroupStatusDeleting, cur.Status, "within the deadline the group keeps waiting")
+
+	backdateNodegroupReap(t, f, "c1", "ng1")
+	sweepNodegroups(t, f)
+
+	desc, err := describeNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleteFailed, aws.StringValue(desc.Nodegroup.Status))
+	require.NotEmpty(t, desc.Nodegroup.Health.Issues)
+	issue := desc.Nodegroup.Health.Issues[0]
+	assert.Equal(t, eks.NodegroupIssueCodeNodeTerminationFailure, aws.StringValue(issue.Code))
+	assert.Contains(t, aws.StringValue(issue.Message), stuck)
+	assert.Contains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
+
+	f.worker.mu.Lock()
+	f.worker.stillLive = nil
+	f.worker.mu.Unlock()
+	_, err = deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
+}

spinifex/domains/eks/cluster/nodegroup_profile.go

@@ -0,0 +1,174 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+const (
+	nodegroupProfilePrefix = "eks-"
+
+	profileClusterTagKey   = "spinifex:eks-cluster"
+	profileNodegroupTagKey = "spinifex:eks-nodegroup"
+	profileIDTagKey        = "spinifex:eks-profile-id"
+)
+
+// nodeProfileForLaunch returns the ARN of the instance profile a worker of rec
+// launches with. A record that names no profile of its own predates per-group
+// profiles and keeps the role-named one it already uses.
+func (s *EKSServiceImpl) nodeProfileForLaunch(accountID string, rec *NodegroupRecord) (string, error) {
+	if rec.InstanceProfileName == "" {
+		return s.ensureNodeInstanceProfile(accountID, rec.NodeRole)
+	}
+	return s.ensureNodegroupProfile(accountID, rec)
+}
+
+func nodegroupProfileTags(rec *NodegroupRecord) []*iam.Tag {
+	return []*iam.Tag{
+		{Key: aws.String(profileClusterTagKey), Value: aws.String(rec.ClusterName)},
+		{Key: aws.String(profileNodegroupTagKey), Value: aws.String(rec.Name)},
+		{Key: aws.String(profileIDTagKey), Value: aws.String(rec.InstanceProfileID)},
+	}
+}
+
+// profileOwnedBy reports whether p carries the ownership tags of this exact
+// incarnation of rec.
+func profileOwnedBy(p *iam.InstanceProfile, rec *NodegroupRecord) bool {
+	have := map[string]string{}
+	for _, t := range p.Tags {
+		have[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
+	}
+	for _, want := range nodegroupProfileTags(rec) {
+		if have[aws.StringValue(want.Key)] != aws.StringValue(want.Value) {
+			return false
+		}
+	}
+	return true
+}
+
+func roleNameFromARN(roleARN string) string {
+	return roleARN[strings.LastIndex(roleARN, "/")+1:]
+}
+
+// ensureNodegroupProfile finds or creates the instance profile recorded for
+// rec, tagged as its own, and makes sure it holds the node role. Concurrent
+// worker launches converge on the one profile.
+func (s *EKSServiceImpl) ensureNodegroupProfile(accountID string, rec *NodegroupRecord) (string, error) {
+	roleName, err := s.resolveNodeRoleName(accountID, rec.NodeRole)
+	if err != nil {
+		return "", err
+	}
+	name := rec.InstanceProfileName
+	ens := s.iamEnsurer()
+
+	got, err := ens.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String(name)})
+	if err != nil {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
+			return "", fmt.Errorf("get instance profile %q: %w", name, err)
+		}
+		_, cerr := ens.CreateInstanceProfile(accountID, &iam.CreateInstanceProfileInput{
+			InstanceProfileName: aws.String(name),
+			Tags:                nodegroupProfileTags(rec),
+		})
+		if cerr != nil {
+			// A racing launch may have created it first; re-read and converge.
+			if code, _ := awserrors.ResolveErrorCode(cerr); code != awserrors.ErrorIAMEntityAlreadyExists {
+				return "", fmt.Errorf("create instance profile %q: %w", name, cerr)
+			}
+		}
+		got, err = ens.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String(name)})
+		if err != nil {
+			return "", fmt.Errorf("re-get instance profile %q: %w", name, err)
+		}
+	}
+	if !profileOwnedBy(got.InstanceProfile, rec) {
+		return "", fmt.Errorf("instance profile %q exists but is not this nodegroup's", name)
+	}
+	return s.attachRoleToProfile(accountID, name, roleName,
+		aws.StringValue(got.InstanceProfile.Arn), len(got.InstanceProfile.Roles) > 0)
+}
+
+// deleteNodegroupProfile removes the profile recorded for rec, and only that
+// one: it must be recorded, tagged for this incarnation, hold exactly the node
+// role and be named by no other record. Anything else is left alone.
+func (s *EKSServiceImpl) deleteNodegroupProfile(ctx context.Context, accountID string, kv jetstream.KeyValue, rec *NodegroupRecord) error {
+	name := rec.InstanceProfileName
+	ens := s.iamEnsurer()
+	if name == "" || ens == nil {
+		return nil
+	}
+	got, err := ens.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String(name)})
+	if err != nil {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
+			return nil
+		}
+		return fmt.Errorf("get instance profile %q: %w", name, err)
+	}
+	p := got.InstanceProfile
+
+	if !profileOwnedBy(p, rec) {
+		slog.WarnContext(ctx, "eks: leaving instance profile whose ownership tags do not match its nodegroup",
+			"cluster", rec.ClusterName, "nodegroup", rec.Name, "profile", name)
+		return nil
+	}
+	nodeRole := roleNameFromARN(rec.NodeRole)
+	if len(p.Roles) > 1 || (len(p.Roles) == 1 && aws.StringValue(p.Roles[0].RoleName) != nodeRole) {
+		slog.WarnContext(ctx, "eks: leaving instance profile that holds a role other than the node role",
+			"cluster", rec.ClusterName, "nodegroup", rec.Name, "profile", name)
+		return nil
+	}
+	shared, err := s.profileNamedByOtherRecord(ctx, kv, rec)
+	if err != nil {
+		return err
+	}
+	if shared {
+		slog.WarnContext(ctx, "eks: leaving instance profile another nodegroup record names",
+			"cluster", rec.ClusterName, "nodegroup", rec.Name, "profile", name)
+		return nil
+	}
+
+	if len(p.Roles) == 1 {
+		_, err := ens.RemoveRoleFromInstanceProfile(accountID, &iam.RemoveRoleFromInstanceProfileInput{
+			InstanceProfileName: aws.String(name), RoleName: aws.String(nodeRole),
+		})
+		if err != nil {
+			if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
+				return fmt.Errorf("remove role %q from instance profile %q: %w", nodeRole, name, err)
+			}
+		}
+	}
+	if _, err := ens.DeleteInstanceProfile(accountID, &iam.DeleteInstanceProfileInput{InstanceProfileName: aws.String(name)}); err != nil {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
+			return fmt.Errorf("delete instance profile %q: %w", name, err)
+		}
+	}
+	return nil
+}
+
+// profileNamedByOtherRecord reports whether any nodegroup record in the
+// account other than rec names rec's profile.
+func (s *EKSServiceImpl) profileNamedByOtherRecord(ctx context.Context, kv jetstream.KeyValue, rec *NodegroupRecord) (bool, error) {
+	clusters, err := listClusterNames(ctx, kv)
+	if err != nil {
+		return false, fmt.Errorf("list clusters: %w", err)
+	}
+	for _, cluster := range clusters {
+		recs, err := ListNodegroupRecords(ctx, kv, cluster)
+		if err != nil {
+			return false, fmt.Errorf("list nodegroups of %s: %w", cluster, err)
+		}
+		for _, other := range recs {
+			if other.InstanceProfileName == rec.InstanceProfileName && (other.ClusterName != rec.ClusterName || other.Name != rec.Name) {
+				return true, nil
+			}
+		}
+	}
+	return false, nil
+}

spinifex/domains/eks/cluster/nodegroup_test.go

@@ -86,6 +86,12 @@ type fakeWorkerLauncher struct {
 	// before subsequent calls succeed — simulates a transient launch failure
 	// (QMP timeout, host pressure) that clears on retry.
 	failNextN int
+	// beforeRun, when set, runs at the start of every launch, before it is
+	// recorded, so a test can inspect durable state at launch time.
+	beforeRun func(*ec2.RunInstancesInput)
+	// stillLive lists instances LiveWorkerInstances keeps reporting as live.
+	stillLive []string
+	liveErr   error
 }
 
 var _ WorkerLauncher = (*fakeWorkerLauncher)(nil)
@@ -101,6 +107,9 @@ func (f *fakeWorkerLauncher) RunWorkerInstance(ctx context.Context, input *ec2.R
 func (f *fakeWorkerLauncher) RunWorkerInstanceOnNode(_ context.Context, nodeID string, input *ec2.RunInstancesInput, _ string) (*ec2.Reservation, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
+	if f.beforeRun != nil {
+		f.beforeRun(input)
+	}
 	f.runCalls = append(f.runCalls, input)
 	f.runNodes = append(f.runNodes, nodeID)
 	if f.failNextN > 0 {
@@ -129,6 +138,21 @@ func (f *fakeWorkerLauncher) TerminateWorkerInstances(_ context.Context, ids []s
 	return f.termErr
 }
 
+func (f *fakeWorkerLauncher) LiveWorkerInstances(_ context.Context, ids []string, _ string) ([]string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.liveErr != nil {
+		return nil, f.liveErr
+	}
+	var live []string
+	for _, id := range ids {
+		if slices.Contains(f.stillLive, id) {
+			live = append(live, id)
+		}
+	}
+	return live, nil
+}
+
 // runCount / terminatedCount return the recorded call totals under the lock so
 // concurrent tests can read them safely after joining.
 func (f *fakeWorkerLauncher) runCount() int {
@@ -927,7 +951,7 @@ func TestUpdateNodegroupConfig_CapacityErrorSurfacesCode(t *testing.T) {
 	})
 }
 
-func TestDeleteNodegroup_TerminatesAndIsIdempotent(t *testing.T) {
+func TestDeleteNodegroup_ReapedGroupIsGoneAndRepeatDeleteIsNotFound(t *testing.T) {
 	t.Parallel()
 	f := newEKSServiceFixture(t)
 	seedActiveClusterWithToken(t, f, "c1")
@@ -940,13 +964,16 @@ func TestDeleteNodegroup_TerminatesAndIsIdempotent(t *testing.T) {
 		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1"),
 	}, testAccountID)
 	require.NoError(t, err)
+	_, err = f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
+	require.NoError(t, err)
+	f.svc.WaitNodegroupDeletes()
 	require.Len(t, f.worker.terminateCalls, 1)
 	assert.Len(t, f.worker.terminateCalls[0], 2)
 
 	_, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
 	assert.ErrorIs(t, err, ErrNodegroupNotFound)
 
-	// Second delete → ResourceNotFoundException (record already gone).
+	// Once the reaper has finished the group is gone: ResourceNotFoundException.
 	_, err = f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
 		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1"),
 	}, testAccountID)

spinifex/domains/eks/cluster/service_impl.go

@@ -201,20 +201,26 @@ type WorkerLauncher interface {
 	// host spread. An empty nodeID launches on the local node like RunWorkerInstance.
 	RunWorkerInstanceOnNode(ctx context.Context, nodeID string, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error)
 	TerminateWorkerInstances(ctx context.Context, instanceIDs []string, accountID string) error
+	// LiveWorkerInstances returns the given workers not yet proven terminated or
+	// absent. An undecidable answer is an error, never an empty list.
+	LiveWorkerInstances(ctx context.Context, instanceIDs []string, accountID string) ([]string, error)
 }
 
 // iamEnsurer resolves the IAM ensurer, preferring the test-injected deps.IAM,
 // then the lazy IAMProvider (built at launch time to dodge the daemon-startup
 // NATS-KV race). Returns nil when neither yields a service, so callers fall back
-// to static creds / skip the profile. The provider's concrete service satisfies
-// instanceProfileEnsurer (identical method set to SystemInstanceRoleEnsurer).
+// to static creds / skip the profile.
 func (s *EKSServiceImpl) iamEnsurer() instanceProfileEnsurer {
 	if s.deps.IAM != nil {
 		return s.deps.IAM
 	}
 	if s.deps.IAMProvider != nil {
 		if e := s.deps.IAMProvider(); e != nil {
-			return e
+			// The concrete service has the wider surface; the shared interface
+			// stays narrow so other callers are not forced to grow it.
+			if ep, ok := e.(instanceProfileEnsurer); ok {
+				return ep
+			}
 		}
 	}
 	return nil
@@ -227,6 +233,8 @@ type instanceProfileEnsurer interface {
 	GetInstanceProfile(accountID string, input *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error)
 	CreateInstanceProfile(accountID string, input *iam.CreateInstanceProfileInput) (*iam.CreateInstanceProfileOutput, error)
 	AddRoleToInstanceProfile(accountID string, input *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error)
+	RemoveRoleFromInstanceProfile(accountID string, input *iam.RemoveRoleFromInstanceProfileInput) (*iam.RemoveRoleFromInstanceProfileOutput, error)
+	DeleteInstanceProfile(accountID string, input *iam.DeleteInstanceProfileInput) (*iam.DeleteInstanceProfileOutput, error)
 
 	// Role find-or-create surface for the system-managed control-plane role.
 	// Unlike worker node roles (customer-supplied), the k3s server role is
@@ -276,6 +284,12 @@ type EKSServiceImpl struct {
 	teardownMu      sync.Mutex
 	teardownRunning map[string]struct{}
 	teardownWG      sync.WaitGroup
+	// noDeleteKick, set by tests, leaves a nodegroup delete to the reaper sweep.
+	noDeleteKick bool
+	// workersEndWait and workersEndPoll bound how long one delete pass waits for
+	// ended workers to be proven gone; zero means the defaults.
+	workersEndWait time.Duration
+	workersEndPoll time.Duration
 	// teardownKV, when set by a test, replaces the converger's account bucket.
 	teardownKV func(ctx context.Context, accountID string) (jetstream.KeyValue, error)
 

spinifex/domains/eks/cluster/types.go

@@ -49,6 +49,16 @@ type NodegroupRecord struct {
 	Tags           map[string]string `json:"tags,omitempty"`
 	InstanceIDs    []string          `json:"instanceIds,omitempty"`
 	Health         string            `json:"health,omitempty"`
+	// InstanceProfileID and InstanceProfileName identify the instance profile
+	// EKS made for this incarnation of the nodegroup; both are empty on a record
+	// that predates it.
+	InstanceProfileID   string `json:"instanceProfileId,omitempty"`
+	InstanceProfileName string `json:"instanceProfileName,omitempty"`
+	// DeletingSince, DeleteReapAttempts and LastDeleteReapAttempt pace the reaper that finishes
+	// a DELETING nodegroup.
+	DeletingSince         time.Time `json:"deletingSince,omitzero"`
+	DeleteReapAttempts    int       `json:"deleteReapAttempts,omitempty"`
+	LastDeleteReapAttempt time.Time `json:"lastDeleteReapAttempt,omitzero"`
 	// GPUEnabled/GPUVendor cache whether InstanceTypes includes a GPU family and
 	// which vendor, so worker launches resolve the matching GPU node AMI without
 	// re-scanning InstanceTypes on every launch.

spinifex/domains/eks/cluster/worker_membership_test.go

@@ -56,6 +56,9 @@ func TestListManagedWorkers_DropsWorkersTheNodegroupTerminated(t *testing.T) {
 		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng2"),
 	}, testAccountID)
 	require.NoError(t, err)
+	_, err = f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
+	require.NoError(t, err)
+	f.svc.WaitNodegroupDeletes()
 	got = managedWorkers(t, f, "c1")
 	assert.Len(t, got, 1)
 	for _, id := range ng2.InstanceIDs {

spinifex/domains/elbv2/awsapi/registration.go

@@ -1,17 +1,14 @@
 package awsapi
 
 import (
-	"bytes"
 	"context"
 	"errors"
-	"io"
 	"log/slog"
 	"maps"
 	"net/http"
 	"slices"
 
 	"github.com/aws/aws-sdk-go/service/elbv2"
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	elbv2domain "github.com/mulgadc/spinifex/spinifex/domains/elbv2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsxml "github.com/mulgadc/spinifex/spinifex/foundation/aws/xml"
@@ -217,7 +214,7 @@ func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
 		InventoryName: InventoryName,
-		ResolveAction: resolveAction,
+		ResolveAction: dispatch.ResolveQueryAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeXML,
 		Inventory:     OperationInventory(),
@@ -235,35 +232,11 @@ func OperationInventory() dispatch.Inventory {
 	return dispatch.Inventory{Registered: ActionNames()}
 }
 
-// resolveAction is the one Query Action parser telemetry and dispatch share.
-// It leaves the body readable for the dispatcher and answers "" on any failure.
-func resolveAction(r *http.Request) string {
-	args, err := readQueryArgs(r)
-	if err != nil {
-		return ""
-	}
-	return args["Action"]
-}
-
-// readQueryArgs parses the Query body and rewinds it, so the same bytes can be
-// read again. Any read or parse failure is a malformed query string.
-func readQueryArgs(r *http.Request) (map[string]string, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, err
-	}
-	r.Body = io.NopCloser(bytes.NewReader(body))
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return query.ParseArgs(string(body))
-}
-
 // dispatch parses, scopes and authorizes the request from one parse of its
 // body, then runs the action. Errors are awserrors codes the gateway renders.
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
-	queryArgs, err := readQueryArgs(r)
+	queryArgs, err := dispatch.ReadArgs(r)
 	if err != nil {
 		slog.DebugContext(r.Context(), "ELBv2: malformed query string", "err", err)
 		return errors.New(awserrors.ErrorMalformedQueryString)

spinifex/domains/elbv2/awsapi/registration_test.go

@@ -3,10 +3,6 @@ package awsapi
 import (
 	"context"
 	"encoding/xml"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"strings"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/service/elbv2"
@@ -31,22 +27,6 @@ func TestNewRegistration_RegistersUnderCredentialScopeAndInventoryName(t *testin
 	assert.Empty(t, inv.Unsupported)
 }
 
-func TestResolveAction_ReadsActionAndRewindsBody(t *testing.T) {
-	body := "Action=DescribeLoadBalancers&Version=2015-12-01"
-	r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
-
-	assert.Equal(t, "DescribeLoadBalancers", resolveAction(r))
-
-	again, err := io.ReadAll(r.Body)
-	require.NoError(t, err)
-	assert.Equal(t, body, string(again), "the dispatcher must still see the whole body")
-}
-
-func TestResolveAction_UnparseableBodyIsEmpty(t *testing.T) {
-	r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=%zz"))
-	assert.Empty(t, resolveAction(r))
-}
-
 // A registration with no quota reports the AWS default cap and enforces none.
 func TestDescribeAccountLimits_NoQuotaReportsDefault(t *testing.T) {
 	out, err := actions["DescribeAccountLimits"].dispatch(t.Context(), "DescribeAccountLimits",

spinifex/domains/elbv2/service_impl.go

@@ -3293,66 +3293,12 @@ func (s *ELBv2ServiceImpl) DescribeTags(ctx context.Context, input *elbv2.Descri
 		}
 		arn := *arnPtr
 
-		resourceType, err := elbv2ResourceTypeFromArn(arn)
+		h, notFoundError, found, err := s.resolveTaggable(ctx, arn)
 		if err != nil {
 			return nil, err
 		}
-
-		var (
-			tags          map[string]string
-			ownerAccount  string
-			notFoundError string
-			found         bool
-		)
-		switch resourceType {
-		case resourcearn.ELBv2LoadBalancer:
-			notFoundError = awserrors.ErrorELBv2LoadBalancerNotFound
-			lb, lbErr := s.store.GetLoadBalancerByArn(ctx, arn)
-			if lbErr != nil {
-				slog.ErrorContext(ctx, "DescribeTags: failed to get LB", "arn", arn, "err", lbErr)
-				return nil, errors.New(awserrors.ErrorServerInternal)
-			}
-			if lb != nil {
-				found = true
-				tags = lb.Tags
-				ownerAccount = lb.AccountID
-			}
-		case resourcearn.ELBv2TargetGroup:
-			notFoundError = awserrors.ErrorELBv2TargetGroupNotFound
-			tg, tgErr := s.store.GetTargetGroupByArn(ctx, arn)
-			if tgErr != nil {
-				slog.ErrorContext(ctx, "DescribeTags: failed to get target group", "arn", arn, "err", tgErr)
-				return nil, errors.New(awserrors.ErrorServerInternal)
-			}
-			if tg != nil {
-				found = true
-				tags = tg.Tags
-				ownerAccount = tg.AccountID
-			}
-		case resourcearn.ELBv2Listener:
-			notFoundError = awserrors.ErrorELBv2ListenerNotFound
-			l, lErr := s.store.GetListenerByArn(ctx, arn)
-			if lErr != nil {
-				slog.ErrorContext(ctx, "DescribeTags: failed to get listener", "arn", arn, "err", lErr)
-				return nil, errors.New(awserrors.ErrorServerInternal)
-			}
-			if l != nil {
-				found = true
-				tags = l.Tags
-				ownerAccount = l.AccountID
-			}
-		case resourcearn.ELBv2ListenerRule:
-			notFoundError = awserrors.ErrorELBv2RuleNotFound
-			r, rErr := s.store.GetRuleByArn(ctx, arn)
-			if rErr != nil {
-				slog.ErrorContext(ctx, "DescribeTags: failed to get rule", "arn", arn, "err", rErr)
-				return nil, errors.New(awserrors.ErrorServerInternal)
-			}
-			if r != nil {
-				found = true
-				tags = r.Tags
-				ownerAccount = r.AccountID
-			} else if lArn, isDefault := listenerArnFromDefaultRuleArn(arn); isDefault {
+		if !found {
+			if lArn, isDefault := listenerArnFromDefaultRuleArn(arn); isDefault {
 				// Synthetic default rule: not stored, carries no tags. Resolve via
 				// its parent listener so a controller's post-create rule-tag sync
 				// gets an empty TagDescription instead of an error.
@@ -3363,19 +3309,18 @@ func (s *ELBv2ServiceImpl) DescribeTags(ctx context.Context, input *elbv2.Descri
 				}
 				if l != nil {
 					found = true
-					tags = nil
-					ownerAccount = l.AccountID
+					h = taggable{owner: l.AccountID}
 				}
 			}
 		}
 
-		if !found || ownerAccount != accountID {
+		if !found || h.owner != accountID {
 			return nil, errors.New(notFoundError)
 		}
 
 		descriptions = append(descriptions, &elbv2.TagDescription{
 			ResourceArn: aws.String(arn),
-			Tags:        tagsMapToSDK(tags),
+			Tags:        tagsMapToSDK(h.tags),
 		})
 	}