fix(ec2): relaunch a direct-boot system guest without demanding a root volume

* test: pin guest poweroff handling and stopped-instance volume state

* test: gofmt the poweroff and stopped-volume tests

* test: pin start after detach, one-poll poweroff, reattach of released volumes and the delete refusal

* test: separate the embedded mounter from the recorder fields

* fix: end a guest poweroff as a stop and keep stopped instances' data volumes attached

A clean QEMU exit after the heartbeat saw the guest power off now stops the
instance instead of waiting for crash detection. Unmount no longer releases
data volumes. DetachVolume on a stopped instance edits the record, then
releases the volume. Starting an instance stopped on an older build drops data
volumes that are attached elsewhere or deleted and fails on a missing boot
volume.

* fix: keep the stopped-instance detach inside the instance service file and format start reconcile

* docs: describe guest poweroff, stopped-instance volumes and detach on a stopped instance

* fix: allow detaching the root volume of a stopped instance and refuse a start with no root volume as AWS does

* feat: attach volumes to a stopped instance and fix root-detach start fixtures

* docs: describe attach and root detach on a stopped instance

* test: Unmount seals only and leaves every attached volume in-use

Changed with the person's explicit approval, relayed by dev-design on 11-03.
EBS root and data volumes persist across stop on AWS, so stop and crash
unmount no longer release any attached volume.

* docs: note that the EFI variable store cannot be detached from a stopped instance

* test: unmount contract tests expect no volume state write on stop

* test: order imports in stopped-volume daemon tests

* refactor: drop the unused volume state updater from the volume mounter

.github/workflows/codex-review.yml

@@ -0,0 +1,21 @@
+# Codex review gate: calls bluebottle's reusable workflows from main, so a PR cannot change its own review.
+name: codex-review
+
+on:
+  pull_request_target:
+    branches: [dev]
+    types: [opened, synchronize, reopened, ready_for_review, edited]
+  issue_comment:
+    types: [created]
+
+permissions: {}
+
+jobs:
+  review:
+    if: github.event_name == 'pull_request_target'
+    uses: mulgadc/bluebottle/.github/workflows/codex-pr-review.yml@main
+    secrets: inherit
+  waive:
+    if: github.event_name == 'issue_comment' && github.event.issue.pull_request
+    uses: mulgadc/bluebottle/.github/workflows/codex-waive.yml@main
+    secrets: inherit

.github/workflows/e2e-baremetal.yml

@@ -71,9 +71,16 @@ jobs:
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ inputs.cell || '20' }}-${{ github.run_id }}
       SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs || format('ci.run_id={0},ci.workflow={1},ci.cell={2},ci.matrix=baremetal/pxe/real-hw', github.run_id, github.workflow, inputs.cell || '20') }}
     steps:
+      # A cancelled job's root steps outlive it, since the runner cannot reach
+      # them once it kills sudo, and would keep writing here and to the cache.
+      # A rootfs killed before bin/bash exists is one the build refuses to reuse.
       - name: Clean Workspace
         run: |
+          sudo systemctl stop spinifex-bm-pxe.scope spinifex-bm-e2e.scope 2>/dev/null || true
           sudo umount -R -l "${BM_ISO_CACHE}/rootfs" 2>/dev/null || true
+          if [ -d "${BM_ISO_CACHE}/rootfs" ] && [ ! -x "${BM_ISO_CACHE}/rootfs/bin/bash" ]; then
+            sudo rm -rf --one-file-system "${BM_ISO_CACHE}/rootfs"
+          fi
           if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
           sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
@@ -113,10 +120,15 @@ jobs:
             "$GITHUB_WORKSPACE" \
             "${RUNNER_TEMP}/spinifex-tests.tar" \
             "${RUNNER_TEMP}/bm-bins"
+          # Built here, as the runner user, so the PXE build finds it current.
+          make -C "$GITHUB_WORKSPACE" build-microvm-image
 
       - name: Build PXE Artifacts
         run: |
-          sudo make -C mulga/scripts/iso-builder \
+          # -o build-spinifex: the binaries come from Build Artifacts, and a
+          # rebuild here would write root-owned files into the workspace.
+          sudo systemd-run --scope --collect --quiet --unit=spinifex-bm-pxe -- \
+            make -C mulga/scripts/iso-builder -o build-spinifex \
             build-rootfs build-squashfs build-initramfs \
             CACHE="${BM_ISO_CACHE}" \
             SPINIFEX_BIN="${RUNNER_TEMP}/bm-bins/spx" \
@@ -124,6 +136,10 @@ jobs:
             SPINIFEX_SRC="${GITHUB_WORKSPACE}" \
             VIPERBLOCK_SRC="${RUNNER_TEMP}/bm-bins/viperblock" \
             VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always --exclude '*-dev' 2>/dev/null || echo dev)"
+          if root_owned="$(find "$GITHUB_WORKSPACE" -user root -print -quit)" && [ -n "$root_owned" ]; then
+            echo "::error::the PXE build wrote root-owned files into the workspace, e.g. $root_owned"
+            exit 1
+          fi
 
       - name: Write ePDU credentials
         run: |
@@ -195,13 +211,18 @@ jobs:
           done
           export BM_TEST_TIMEOUT
           echo "suites=${BM_SUITES} test-timeout=${BM_TEST_TIMEOUT}"
-          sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
+          sudo -E systemd-run --scope --collect --quiet --unit=spinifex-bm-e2e -- \
+            mulga/scripts/baremetal/run-baremetal-e2e.sh \
             --vmlinuz   "${BM_ISO_CACHE}/vmlinuz" \
             --initrd    "${BM_ISO_CACHE}/initrd.img" \
             --squashfs  "${BM_ISO_CACHE}/spinifex.squashfs" \
             --tests-tar "${RUNNER_TEMP}/spinifex-tests.tar" \
             2>&1 | tee "${ARTIFACT_DIR}/test-baremetal.log"
 
+      - name: Stop this job's root steps
+        if: always()
+        run: sudo systemctl stop spinifex-bm-pxe.scope spinifex-bm-e2e.scope 2>/dev/null || true
+
       - name: Generate JUnit report
         if: always()
         run: |

architecture/layering/allowlist_test.go

@@ -15,9 +15,6 @@ var allowlist = []allowance{
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/igw", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/image", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/instance", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
-	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/key", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
-	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/launchtemplate", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
-	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/placementgroup", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/routetable", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/snapshot", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/volume", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},

cmd/lambda-agent/package_testlink.go

@@ -0,0 +1,276 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// Dev-only code delivery for the test path: the host attaches the ZIP as a
+// read-only disk and passes its exact size and SHA-256 through fw_cfg. This
+// is not the production delivery, and these bounds are not AWS limits.
+const (
+	entryPackageSize   = "test-package-size"
+	entryPackageSHA256 = "test-package-sha256"
+
+	devMaxZipBytes    = 16 << 20
+	devMaxUnzipBytes  = 64 << 20
+	devMaxEntries     = 1024
+	packageDevice     = "/dev/vda"
+	taskTmpfsSizeOpts = "mode=0755,size=72m"
+)
+
+type packageSpec struct {
+	size   int64
+	sha256 [sha256.Size]byte
+}
+
+type unpackLimits struct {
+	entries int
+	bytes   int64
+}
+
+var devLimits = unpackLimits{entries: devMaxEntries, bytes: devMaxUnzipBytes}
+
+// readPackageSpec reads the expected size and SHA-256 from fw_cfg; it must
+// run before fw_cfg is sealed.
+func readPackageSpec(fwCfgDir string) (packageSpec, error) {
+	var spec packageSpec
+	read := func(name string) (string, error) {
+		b, err := os.ReadFile(filepath.Join(fwCfgDir, name, "raw"))
+		if err != nil {
+			return "", fmt.Errorf("read %s: %w", name, err)
+		}
+		return strings.TrimSpace(string(b)), nil
+	}
+	s, err := read(entryPackageSize)
+	if err != nil {
+		return spec, err
+	}
+	spec.size, err = strconv.ParseInt(s, 10, 64)
+	if err != nil || spec.size <= 0 || spec.size > devMaxZipBytes {
+		return spec, fmt.Errorf("package size %q outside 1..%d bytes", s, devMaxZipBytes)
+	}
+	h, err := read(entryPackageSHA256)
+	if err != nil {
+		return spec, err
+	}
+	b, err := hex.DecodeString(h)
+	if err != nil || len(b) != sha256.Size {
+		return spec, errors.New("package sha256 is not 64 hex digits")
+	}
+	copy(spec.sha256[:], b)
+	return spec, nil
+}
+
+// taskPackage puts the package under dir: a fresh nosuid,nodev tmpfs, made
+// read-only once the files are in place.
+type taskPackage struct {
+	device     string
+	dir        string
+	loadDriver func() error
+	mountDir   func(dir string) error
+	sealDir    func(dir string) error
+}
+
+func devTaskPackage() taskPackage {
+	return taskPackage{device: packageDevice, dir: taskRoot, loadDriver: loadBlockDriver, mountDir: mountTaskDir, sealDir: sealTaskDir}
+}
+
+func (p taskPackage) install(spec packageSpec) error {
+	if err := p.loadDriver(); err != nil {
+		return fmt.Errorf("load block driver: %w", err)
+	}
+	dev, err := os.Open(p.device)
+	if err != nil {
+		return fmt.Errorf("open package disk: %w", err)
+	}
+	defer dev.Close()
+	if err := p.mountDir(p.dir); err != nil {
+		return fmt.Errorf("mount %s: %w", p.dir, err)
+	}
+	if err := unpack(dev, spec, p.dir, devLimits); err != nil {
+		return err
+	}
+	if err := p.sealDir(p.dir); err != nil {
+		return fmt.Errorf("make %s read-only: %w", p.dir, err)
+	}
+	return nil
+}
+
+// unpack reads exactly spec.size bytes once, checks their SHA-256, then
+// validates every entry before writing any of them under dir.
+func unpack(r io.ReaderAt, spec packageSpec, dir string, lim unpackLimits) error {
+	buf := make([]byte, spec.size)
+	if _, err := io.ReadFull(io.NewSectionReader(r, 0, spec.size), buf); err != nil {
+		return fmt.Errorf("read package: %w", err)
+	}
+	if sha256.Sum256(buf) != spec.sha256 {
+		return errors.New("package sha256 mismatch")
+	}
+	zr, err := zip.NewReader(bytes.NewReader(buf), spec.size)
+	if err != nil {
+		return fmt.Errorf("open package: %w", err)
+	}
+	if len(zr.File) > lim.entries {
+		return fmt.Errorf("package has %d entries, limit %d", len(zr.File), lim.entries)
+	}
+	names := make([]string, len(zr.File))
+	var declared int64
+	for i, f := range zr.File {
+		name, err := entryName(f.Name)
+		if err != nil {
+			return err
+		}
+		names[i] = name
+		switch mode := f.Mode(); {
+		case mode.IsDir():
+		case mode.IsRegular():
+			// Compared against what remains, so a huge size cannot wrap the sum.
+			if f.UncompressedSize64 > uint64(lim.bytes-declared) {
+				return fmt.Errorf("%q: unpacked size over the %d-byte limit", f.Name, lim.bytes)
+			}
+			declared += int64(f.UncompressedSize64)
+		default:
+			return fmt.Errorf("%q: %s entries are refused", f.Name, mode.Type())
+		}
+	}
+
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		return err
+	}
+	defer root.Close()
+	var written int64
+	for i, f := range zr.File {
+		n, err := extract(root, names[i], f, lim.bytes-written)
+		written += n
+		if err != nil {
+			return fmt.Errorf("%q: %w", f.Name, err)
+		}
+	}
+	return nil
+}
+
+// extract writes one entry, refusing it if its actual bytes exceed budget
+// whatever its header claims.
+func extract(root *os.Root, name string, f *zip.File, budget int64) (int64, error) {
+	if f.Mode().IsDir() {
+		return 0, mkdirs(root, name)
+	}
+	if d := path.Dir(name); d != "." {
+		if err := mkdirs(root, d); err != nil {
+			return 0, err
+		}
+	}
+	perm := os.FileMode(0o644)
+	if f.Mode()&0o111 != 0 {
+		perm = 0o755
+	}
+	in, err := f.Open()
+	if err != nil {
+		return 0, err
+	}
+	defer in.Close()
+	out, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
+	if err != nil {
+		return 0, err
+	}
+	// The umask applies at create; fchmod does not, so the mode is exact.
+	if err := out.Chmod(perm); err != nil {
+		out.Close()
+		return 0, err
+	}
+	n, err := io.Copy(out, io.LimitReader(in, budget+1))
+	if cerr := out.Close(); err == nil {
+		err = cerr
+	}
+	if err == nil && n > budget {
+		err = errors.New("unpacked bytes over the limit")
+	}
+	return n, err
+}
+
+// mkdirs creates each component of dir as mode 0755 whatever the umask. An
+// existing component must be a real directory, never a symlink.
+func mkdirs(root *os.Root, dir string) error {
+	cur := ""
+	for _, part := range strings.Split(dir, "/") {
+		cur = path.Join(cur, part)
+		if err := root.Mkdir(cur, 0o755); err != nil {
+			if !errors.Is(err, fs.ErrExist) {
+				return err
+			}
+			fi, err := root.Lstat(cur)
+			if err != nil {
+				return err
+			}
+			if !fi.IsDir() {
+				return fmt.Errorf("%q exists and is not a directory", cur)
+			}
+		}
+		if err := root.Chmod(cur, 0o755); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// entryName returns a clean relative path, refusing anything that could name
+// a location outside the task root.
+func entryName(name string) (string, error) {
+	if name == "" || strings.ContainsAny(name, "\\\x00") || path.IsAbs(name) {
+		return "", fmt.Errorf("%q: entry name refused", name)
+	}
+	c := path.Clean(name)
+	if c == "." || c == ".." || strings.HasPrefix(c, "../") {
+		return "", fmt.Errorf("%q: entry name refused", name)
+	}
+	return c, nil
+}
+
+// loadBlockDriver loads virtio_blk, which the image ships but init does not
+// load, then waits briefly for the package disk to appear.
+func loadBlockDriver() error {
+	if err := exec.Command("/sbin/modprobe", "virtio_blk").Run(); err != nil {
+		var uts unix.Utsname
+		if uerr := unix.Uname(&uts); uerr != nil {
+			return uerr
+		}
+		ko := "/lib/modules/" + unix.ByteSliceToString(uts.Release[:]) + "/kernel/drivers/block/virtio_blk.ko"
+		if err := exec.Command("/sbin/insmod", ko).Run(); err != nil {
+			return fmt.Errorf("modprobe and insmod virtio_blk failed: %w", err)
+		}
+	}
+	for range 50 {
+		if _, err := os.Stat(packageDevice); err == nil {
+			return nil
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	return fmt.Errorf("%s did not appear", packageDevice)
+}
+
+func mountTaskDir(dir string) error {
+	return unix.Mount("tmpfs", dir, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, taskTmpfsSizeOpts)
+}
+
+func sealTaskDir(dir string) error {
+	return unix.Mount("", dir, "", unix.MS_REMOUNT|unix.MS_RDONLY|unix.MS_NOSUID|unix.MS_NODEV, "")
+}

cmd/lambda-agent/package_testlink_test.go

@@ -0,0 +1,209 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"hash/crc32"
+	"io/fs"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testZip builds a ZIP; an entry named bootstrap is marked executable.
+func testZip(t *testing.T, files map[string]string) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	names := make([]string, 0, len(files))
+	for n := range files {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	for _, n := range names {
+		h := &zip.FileHeader{Name: n, Method: zip.Deflate}
+		h.SetMode(0o644)
+		if n == "bootstrap" {
+			h.SetMode(0o755)
+		}
+		w, err := zw.CreateHeader(h)
+		require.NoError(t, err)
+		_, err = w.Write([]byte(files[n]))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+// rawZip writes entries with headers set by the caller, so a test can claim
+// sizes or modes the data does not have.
+func rawZip(t *testing.T, entries ...*zip.FileHeader) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for _, h := range entries {
+		w, err := zw.CreateRaw(h)
+		require.NoError(t, err)
+		_, err = w.Write([]byte("x"))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+func specFor(b []byte) packageSpec {
+	return packageSpec{size: int64(len(b)), sha256: sha256.Sum256(b)}
+}
+
+// Not parallel: the umask is process-wide, so the test sets a restrictive one
+// and restores it.
+func TestUnpackInstallsFilesWithFixedModes(t *testing.T) {
+	old := syscall.Umask(0o077)
+	t.Cleanup(func() { syscall.Umask(old) })
+	b := testZip(t, map[string]string{"bootstrap": "#!/bin/sh\n", "lib/data.txt": "data"})
+	dir := t.TempDir()
+	require.NoError(t, unpack(bytes.NewReader(b), specFor(b), dir, devLimits))
+
+	pfi, err := os.Stat(filepath.Join(dir, "lib"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o755), pfi.Mode().Perm(), "implicit parent dir")
+
+	fi, err := os.Stat(filepath.Join(dir, "bootstrap"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o755), fi.Mode().Perm(), "executable")
+	fi, err = os.Stat(filepath.Join(dir, "lib", "data.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o644), fi.Mode().Perm())
+	got, err := os.ReadFile(filepath.Join(dir, "lib", "data.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, "data", string(got))
+}
+
+func TestUnpackRefusesBeforeWritingAnything(t *testing.T) {
+	symlink := &zip.FileHeader{Name: "link", Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: 1}
+	symlink.SetMode(fs.ModeSymlink | 0o777)
+	fifo := &zip.FileHeader{Name: "pipe", Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: 1}
+	fifo.SetMode(fs.ModeNamedPipe | 0o644)
+	// The second size would wrap a plain running sum back under the limit.
+	big := func(name string, n uint64) *zip.FileHeader {
+		return &zip.FileHeader{Name: name, Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: n}
+	}
+	cases := map[string]struct {
+		zip  []byte
+		want string
+	}{
+		"parent traversal": {testZip(t, map[string]string{"a": "ok", "../escape": "x"}), "entry name refused"},
+		"nested traversal": {testZip(t, map[string]string{"a": "ok", "lib/../../escape": "x"}), "entry name refused"},
+		"absolute path":    {testZip(t, map[string]string{"a": "ok", "/etc/escape": "x"}), "entry name refused"},
+		"backslash":        {testZip(t, map[string]string{"a": "ok", `..\escape`: "x"}), "entry name refused"},
+		"symlink":          {rawZip(t, big("a", 1), symlink), "entries are refused"},
+		"named pipe":       {rawZip(t, big("a", 1), fifo), "entries are refused"},
+		"declared size":    {rawZip(t, big("a", devMaxUnzipBytes+1)), "over the"},
+		"wrapping sizes":   {rawZip(t, big("a", devMaxUnzipBytes), big("b", math.MaxUint64)), "over the"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := t.TempDir()
+			err := unpack(bytes.NewReader(tc.zip), specFor(tc.zip), dir, devLimits)
+			assert.ErrorContains(t, err, tc.want)
+			left, rerr := os.ReadDir(dir)
+			require.NoError(t, rerr)
+			assert.Empty(t, left, "files written before the refusal")
+		})
+	}
+}
+
+func TestUnpackRefusesTooManyEntries(t *testing.T) {
+	b := testZip(t, map[string]string{"a": "1", "b": "2", "c": "3"})
+	err := unpack(bytes.NewReader(b), specFor(b), t.TempDir(), unpackLimits{entries: 2, bytes: devMaxUnzipBytes})
+	assert.ErrorContains(t, err, "limit 2")
+}
+
+// The header claims a size within the limit, but the data is longer: the
+// actual bytes are refused, whatever the header says.
+func TestUnpackRefusesBytesBeyondTheLimit(t *testing.T) {
+	data := bytes.Repeat([]byte("a"), 64)
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	h := &zip.FileHeader{Name: "a", Method: zip.Store, CRC32: crc32.ChecksumIEEE(data), CompressedSize64: uint64(len(data)), UncompressedSize64: 8}
+	w, err := zw.CreateRaw(h)
+	require.NoError(t, err)
+	_, err = w.Write(data)
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	b := buf.Bytes()
+
+	dir := t.TempDir()
+	err = unpack(bytes.NewReader(b), specFor(b), dir, unpackLimits{entries: 4, bytes: 16})
+	require.Error(t, err)
+	fi, serr := os.Stat(filepath.Join(dir, "a"))
+	if serr == nil {
+		assert.LessOrEqual(t, fi.Size(), int64(17))
+	}
+}
+
+func TestUnpackChecksSizeAndHash(t *testing.T) {
+	b := testZip(t, map[string]string{"bootstrap": "x"})
+	wrongHash := specFor(b)
+	wrongHash.sha256[0] ^= 1
+	assert.ErrorContains(t, unpack(bytes.NewReader(b), wrongHash, t.TempDir(), devLimits), "sha256 mismatch")
+
+	longer := specFor(b)
+	longer.size++
+	assert.ErrorContains(t, unpack(bytes.NewReader(b), longer, t.TempDir(), devLimits), "read package")
+}
+
+func TestReadPackageSpecBounds(t *testing.T) {
+	sum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	cases := map[string]struct {
+		size, hash string
+		want       string
+	}{
+		"valid":     {"10", sum, ""},
+		"zero":      {"0", sum, "outside"},
+		"too large": {"16777217", sum, "outside"},
+		"not hex":   {"10", "zz", "64 hex digits"},
+		"short":     {"10", sum[:62], "64 hex digits"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := t.TempDir()
+			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash} {
+				require.NoError(t, os.MkdirAll(filepath.Join(dir, n), 0o700))
+				require.NoError(t, os.WriteFile(filepath.Join(dir, n, "raw"), []byte(v), 0o600))
+			}
+			spec, err := readPackageSpec(dir)
+			if tc.want == "" {
+				require.NoError(t, err)
+				assert.Equal(t, int64(10), spec.size)
+				return
+			}
+			assert.ErrorContains(t, err, tc.want)
+		})
+	}
+}
+
+// A pre-existing symlink where a directory belongs is refused, not followed
+// or chmodded.
+func TestMkdirsRefusesSymlinkComponent(t *testing.T) {
+	dir := t.TempDir()
+	outside := t.TempDir()
+	require.NoError(t, os.Chmod(outside, 0o700))
+	require.NoError(t, os.Symlink(outside, filepath.Join(dir, "lib")))
+	root, err := os.OpenRoot(dir)
+	require.NoError(t, err)
+	defer root.Close()
+	assert.ErrorContains(t, mkdirs(root, "lib/sub"), "not a directory")
+	fi, err := os.Stat(outside)
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o700), fi.Mode().Perm(), "symlink target was chmodded")
+}

cmd/lambda-agent/start_testlink.go

@@ -30,6 +30,7 @@ var handler = flag.String("handler", os.Getenv("_HANDLER"), "function handler pa
 // connection; the carrier is not the guest protocol.
 type testlink struct {
 	startup
+	pkg            taskPackage
 	runtimeAddr    string
 	startBootstrap func(ctx context.Context, env []string) (*exec.Cmd, error)
 }
@@ -37,6 +38,7 @@ type testlink struct {
 func start(ctx context.Context) int {
 	t := testlink{
 		startup:     defaultStartup(),
+		pkg:         devTaskPackage(),
 		runtimeAddr: runtimeAddr,
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			return agent.StartBootstrap(ctx, taskRoot+"/bootstrap", env, os.Stdout, os.Stderr)
@@ -50,11 +52,16 @@ func start(ctx context.Context) int {
 }
 
 func (t testlink) run(parent context.Context) (int, error) {
+	// The package values are fw_cfg entries, so read them before connect seals it.
+	spec, specErr := readPackageSpec(t.fwCfgDir)
 	conn, code, err := t.connect(parent)
 	if err != nil {
-		return code, err
+		return code, errors.Join(err, specErr)
 	}
 	defer conn.Close()
+	if specErr != nil {
+		return exitRefused, specErr
+	}
 	link := hostlink.NewTestStream(conn)
 
 	// The handshake can return before the host refuses the guest, so nothing
@@ -69,6 +76,9 @@ func (t testlink) run(parent context.Context) (int, error) {
 	if parent.Err() != nil {
 		return exitLink, context.Cause(parent)
 	}
+	if err := t.pkg.install(spec); err != nil {
+		return exitRefused, fmt.Errorf("install package: %w", err)
+	}
 
 	srv := runtimeapi.New()
 	ln, err := net.Listen("tcp", t.runtimeAddr)

cmd/lambda-agent/start_testlink_test.go

@@ -5,10 +5,14 @@ package main
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"io"
 	"net/http"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"strconv"
 	"testing"
 	"time"
 
@@ -43,12 +47,31 @@ func TestHelperBootstrap(t *testing.T) {
 type bootstrapStarts struct {
 	n       int
 	started chan struct{}
+	task    string
+	sealed  bool
 }
 
-func (b *bootstrapStarts) testlink(g *guest) testlink {
+// testlink builds a guest whose fw_cfg also carries a controlled package,
+// installed into a temp dir without mounting anything.
+func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) testlink {
+	t.Helper()
+	zipBytes := testZip(t, map[string]string{"bootstrap": "#!/bin/sh\n", "lib/data.txt": "data"})
+	disk := filepath.Join(t.TempDir(), "disk")
+	require.NoError(t, os.WriteFile(disk, zipBytes, 0o600))
+	sum := sha256.Sum256(zipBytes)
+	entries[entryPackageSize] = []byte(strconv.Itoa(len(zipBytes)))
+	entries[entryPackageSHA256] = []byte(hex.EncodeToString(sum[:]))
+	g := newGuest(t, entries)
 	b.started = make(chan struct{}, 1)
+	b.task = t.TempDir()
 	return testlink{
-		startup:     g.startup(),
+		startup: g.startup(),
+		pkg: taskPackage{
+			device: disk, dir: b.task,
+			loadDriver: func() error { return nil },
+			mountDir:   func(string) error { return nil },
+			sealDir:    func(string) error { b.sealed = true; return nil },
+		},
 		runtimeAddr: "127.0.0.1:0",
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			b.n++
@@ -80,7 +103,7 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
 
 	conn := h.guest(t)
 	defer conn.Close()
@@ -109,6 +132,8 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		assert.Error(t, err, "agent answered a second invocation")
 	}
 	assert.Equal(t, 1, b.n)
+	assert.True(t, b.sealed, "task dir not made read-only")
+	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
 }
 
 func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
@@ -117,7 +142,7 @@ func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
 
 	conn := h.guest(t)
 	defer conn.Close()
@@ -142,7 +167,7 @@ func TestTestlinkHostRefusalNeverStartsBootstrap(t *testing.T) {
 	entries[lambdav1.EntryGuestKey], entries[lambdav1.EntryGuestCert] = wrong[lambdav1.EntryGuestKey], wrong[lambdav1.EntryGuestCert]
 	var b bootstrapStarts
 
-	res := wait(t, runTestlinkAsync(ctx, b.testlink(newGuest(t, entries))), "agent to stop")
+	res := wait(t, runTestlinkAsync(ctx, b.testlink(t, entries)), "agent to stop")
 	assert.Equal(t, exitLink, res.code)
 	require.Error(t, res.err)
 	assert.Contains(t, res.err.Error(), "bad certificate")
@@ -155,7 +180,7 @@ func TestTestlinkHostCloseBeforeFrameNeverStartsBootstrap(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
 
 	require.NoError(t, h.guest(t).Close())
 	res := wait(t, done, "agent to stop")
@@ -177,3 +202,45 @@ func TestOneInvocationHonoursCancelledContext(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "r-1", req.RequestID)
 }
+
+// A package that fails its check is refused after authentication, and the
+// bootstrap never starts.
+func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	tl := b.testlink(t, h.entries())
+	tl.pkg.device = filepath.Join(t.TempDir(), "other")
+	require.NoError(t, os.WriteFile(tl.pkg.device, testZip(t, map[string]string{"bootstrap": "x"}), 0o600))
+	done := runTestlinkAsync(ctx, tl)
+
+	conn := h.guest(t)
+	defer conn.Close()
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{RequestID: "r-1"}))
+	res := wait(t, done, "agent to stop")
+	assert.Equal(t, exitRefused, res.code)
+	assert.ErrorContains(t, res.err, "install package")
+	assert.Zero(t, b.n)
+	assert.False(t, b.sealed)
+}
+
+// Missing package values still seal fw_cfg, and the agent refuses before
+// waiting for any frame.
+func TestTestlinkMissingPackageSpecRefuses(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	tl := b.testlink(t, h.entries())
+	require.NoError(t, os.RemoveAll(filepath.Join(tl.fwCfgDir, entryPackageSHA256)))
+
+	res := wait(t, runTestlinkAsync(ctx, tl), "agent to stop")
+	assert.Equal(t, exitRefused, res.code)
+	assert.ErrorContains(t, res.err, "read "+entryPackageSHA256)
+	assert.Zero(t, b.n)
+	_, err := os.Stat(tl.fwCfgRoot)
+	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg not sealed")
+}

contracts/ec2/v1/README.md

@@ -23,7 +23,7 @@ subject shape are wire compatibility. Callers and receivers must use this
 package rather than rebuild those details. `commands_test.go` preserves the
 version-1 JSON shape and subject form.
 
-This package owns shared command payloads and the drain-volume response. It
+This package owns shared command payloads. It
 does not define every action's reply body, authorize the request, store EC2
 state or implement NATS transport. Those remain responsibilities of the EC2
 domain and its runtime composition.

contracts/ec2/v1/commands.go

@@ -29,7 +29,6 @@ type EC2InstanceCommand struct {
 	Attributes                EC2CommandAttributes       `json:"attributes"`
 	AttachVolumeData          *AttachVolumeData          `json:"attach_volume_data,omitempty"`
 	DetachVolumeData          *DetachVolumeData          `json:"detach_volume_data,omitempty"`
-	DrainVolumeData           *DrainVolumeData           `json:"drain_volume_data,omitempty"`
 	AttachENIData             *AttachENIData             `json:"attach_eni_data,omitempty"`
 	DetachENIData             *DetachENIData             `json:"detach_eni_data,omitempty"`
 	IamProfileAssociationData *IamProfileAssociationData `json:"iam_profile_association_data,omitempty"`
@@ -45,7 +44,6 @@ type EC2CommandAttributes struct {
 	StartInstance               bool `json:"start_instance"`
 	AttachVolume                bool `json:"attach_volume"`
 	DetachVolume                bool `json:"detach_volume"`
-	DrainVolume                 bool `json:"drain_volume,omitempty"`
 	RebootInstance              bool `json:"reboot_instance"`
 	AttachENI                   bool `json:"attach_eni"`
 	DetachENI                   bool `json:"detach_eni"`
@@ -69,33 +67,6 @@ type DetachVolumeData struct {
 	Force    bool   `json:"force,omitempty"`
 }
 
-// DrainVolumeData carries parameters for a drain-volume command, which flushes
-// the volume's in-flight writes to S3 before a snapshot reads them. The command
-// is addressed to the instance the volume is attached to because the drain
-// socket only exists on the node hosting it.
-type DrainVolumeData struct {
-	VolumeID string `json:"volume_id"`
-}
-
-const (
-	// DrainVolumeStatusDrained is the ack a node returns once the volume's
-	// in-flight writes have reached S3.
-	DrainVolumeStatusDrained = "drained"
-
-	// DrainVolumeStatusNotRunning is the ack a node returns when it still holds
-	// the instance but the VM is not running, so no process is writing to the
-	// volume and its sealed checkpoint is already current. An attachment record
-	// outlives the writer — stop deliberately leaves boot volumes attached — so
-	// the caller cannot tell this from the record alone.
-	DrainVolumeStatusNotRunning = "not-running"
-)
-
-// DrainVolumeResponse is the reply to a drain-volume command.
-type DrainVolumeResponse struct {
-	VolumeID string `json:"volume_id"`
-	Status   string `json:"status"`
-}
-
 // AttachENIData carries parameters for an attach-network-interface command.
 type AttachENIData struct {
 	NetworkInterfaceID string `json:"network_interface_id"`

contracts/ec2/v1/commands_test.go

@@ -16,7 +16,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			StartInstance:               true,
 			AttachVolume:                true,
 			DetachVolume:                true,
-			DrainVolume:                 true,
 			RebootInstance:              true,
 			AttachENI:                   true,
 			DetachENI:                   true,
@@ -28,7 +27,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		},
 		AttachVolumeData:          &AttachVolumeData{VolumeID: "vol-1", Device: "/dev/sdf"},
 		DetachVolumeData:          &DetachVolumeData{VolumeID: "vol-2", Device: "/dev/sdg", Force: true},
-		DrainVolumeData:           &DrainVolumeData{VolumeID: "vol-3"},
 		AttachENIData:             &AttachENIData{NetworkInterfaceID: "eni-1", DeviceIndex: 2},
 		DetachENIData:             &DetachENIData{AttachmentID: "eni-attach-1", Force: true},
 		IamProfileAssociationData: &IamProfileAssociationData{InstanceProfileArn: "arn:aws:iam::123456789012:instance-profile/example"},
@@ -47,7 +45,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			"start_instance": true,
 			"attach_volume": true,
 			"detach_volume": true,
-			"drain_volume": true,
 			"reboot_instance": true,
 			"attach_eni": true,
 			"detach_eni": true,
@@ -59,7 +56,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		},
 		"attach_volume_data": {"volume_id": "vol-1", "device": "/dev/sdf"},
 		"detach_volume_data": {"volume_id": "vol-2", "device": "/dev/sdg", "force": true},
-		"drain_volume_data": {"volume_id": "vol-3"},
 		"attach_eni_data": {"network_interface_id": "eni-1", "device_index": 2},
 		"detach_eni_data": {"attachment_id": "eni-attach-1", "force": true},
 		"iam_profile_association_data": {"instance_profile_arn": "arn:aws:iam::123456789012:instance-profile/example"},
@@ -69,16 +65,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 	}`, string(data))
 }
 
-func TestDrainVolumeResponseJSONContract(t *testing.T) {
-	data, err := json.Marshal(DrainVolumeResponse{
-		VolumeID: "vol-1",
-		Status:   DrainVolumeStatusDrained,
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"volume_id":"vol-1","status":"drained"}`, string(data))
-	require.Equal(t, "not-running", DrainVolumeStatusNotRunning)
-}
-
 func TestInstanceCommandSubjects(t *testing.T) {
 	require.Equal(t, "ec2.cmd.i-0123456789abcdef0", InstanceCommandSubject("i-0123456789abcdef0"))
 	require.Equal(t, "ec2.cmd.*", InstanceCommandSubjectWildcard)

contracts/operator/v1/admin.go

@@ -25,6 +25,10 @@ const (
 	DeletionStateRunning   = "RUNNING"
 	DeletionStateCompleted = "COMPLETED"
 	DeletionStateFailed    = "FAILED"
+
+	// DeletionStateExercised ends a non-finalizing run: the account is left
+	// TERMINATING and a later DeleteAccount takes the same operation over.
+	DeletionStateExercised = "EXERCISED"
 )
 
 // CreateAccountRequest is the /admin/CreateAccount request body.
@@ -52,6 +56,10 @@ type DeleteAccountRequest struct {
 	ClientToken string `json:"clientToken"`
 	Force       bool   `json:"force,omitempty"`
 	DryRun      bool   `json:"dryRun,omitempty"`
+
+	// NonFinalizing quiesces the account, cleans the quiesced EC2 platform
+	// kinds and stops. No legacy stage runs and the account record stays.
+	NonFinalizing bool `json:"nonFinalizing,omitempty"`
 }
 
 // DescribeAccountDeletionRequest is the /admin/DescribeAccountDeletion body.
@@ -76,8 +84,8 @@ type ListAccountsResponse struct {
 }
 
 // IAMQuiescence is the coordinator's recorded view of IAM quiescence for one
-// fence. Recording only: it admits nothing, and QuiescedScope stays empty until
-// a final inventory re-read exists to catch a late write.
+// fence. Recording only: it admits nothing. QuiescedScope is filled only by a run
+// that guarantees a final inventory re-read, and holds as of EvaluatedAt.
 type IAMQuiescence struct {
 	OperationID   string            `json:"operationId"`
 	Fence         uint64            `json:"fence"`
@@ -136,3 +144,67 @@ const (
 	ReasonOwnerInventoryIncomplete     = "owner-inventory-incomplete"
 	ReasonFinalInventoryRereadNotWired = "final-inventory-reread-not-wired"
 )
+
+// ExerciseResult is what a non-finalizing run did after IAM quiescence.
+type ExerciseResult struct {
+	// Skipped is the fixed reason no cleanup ran; Cleanup is then absent.
+	Skipped    string           `json:"skipped,omitempty"`
+	Cleanup    *PlatformCleanup `json:"cleanup,omitempty"`
+	Accounting []KindAccounting `json:"accounting,omitempty"`
+}
+
+// PlatformCleanup is the EC2 platform cleanup result. It carries resource IDs,
+// fixed reasons and counts only, never names, tags, key material or data.
+type PlatformCleanup struct {
+	Kinds  []CleanupKind  `json:"kinds"`
+	Items  []CleanupItem  `json:"items,omitempty"`
+	Counts []CleanupCount `json:"counts,omitempty"`
+}
+
+// CleanupKind is one kind's scan status after the cleanup's own re-scan.
+type CleanupKind struct {
+	Kind              string   `json:"kind"`
+	Status            string   `json:"status"`
+	IncompleteReasons []string `json:"incompleteReasons,omitempty"`
+}
+
+// CleanupItem is one resource by its ID with what the cleanup did to it.
+type CleanupItem struct {
+	ResourceID  string `json:"resourceId"`
+	Kind        string `json:"kind"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	MemberCount int    `json:"memberCount,omitempty"`
+}
+
+// CleanupCount is the number of records of one class that have no public ID.
+type CleanupCount struct {
+	Kind        string `json:"kind"`
+	Class       string `json:"class"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	Count       int    `json:"count"`
+}
+
+// KindAccounting is the final inventory re-read held against the cleanup result
+// for one kind. Quiescence is not rewritten from it.
+type KindAccounting struct {
+	Kind      string `json:"kind"`
+	Status    string `json:"status"`
+	Reason    string `json:"reason,omitempty"`
+	Remaining int    `json:"remaining"`
+}
+
+// Accounting statuses and fixed reasons of a KindAccounting.
+const (
+	AccountingAccounted   = "accounted"
+	AccountingUnaccounted = "unaccounted"
+
+	AccountingReasonRereadIncomplete = "reread-incomplete"
+	AccountingReasonUnnamedRemainder = "unnamed-remainder"
+)
+
+// Fixed reasons an exercise ran no cleanup.
+const (
+	ExerciseSkippedNoQuiescedKind = "no-quiesced-kind"
+)

contracts/operator/v1/admin_test.go

@@ -43,6 +43,27 @@ func TestAdminWire_DeletionStates(t *testing.T) {
 	require.Equal(t, "RUNNING", DeletionStateRunning)
 	require.Equal(t, "COMPLETED", DeletionStateCompleted)
 	require.Equal(t, "FAILED", DeletionStateFailed)
+	require.Equal(t, "EXERCISED", DeletionStateExercised)
+}
+
+func TestAdminWire_NonFinalizingDeleteRequest(t *testing.T) {
+	goldenJSON(t, DeleteAccountRequest{AccountID: "000000000042", AccountName: "acme", ClientToken: "tok-3", NonFinalizing: true},
+		`{"accountId":"000000000042","accountName":"acme","clientToken":"tok-3","nonFinalizing":true}`)
+}
+
+func TestAdminWire_ExerciseResult(t *testing.T) {
+	goldenJSON(t, ExerciseResult{
+		Cleanup: &PlatformCleanup{
+			Kinds:  []CleanupKind{{Kind: "key-pair", Status: "observed-clean-at-scan-time"}},
+			Items:  []CleanupItem{{ResourceID: "pg-1", Kind: "placement-group", Disposition: "blocked", Reason: "instance-members", MemberCount: 1}},
+			Counts: []CleanupCount{{Kind: "key-pair", Class: "public-key-object-without-metadata", Disposition: "deleted", Count: 2}},
+		},
+		Accounting: []KindAccounting{{Kind: "placement-group", Status: AccountingAccounted, Remaining: 1}},
+	}, `{"cleanup":{"kinds":[{"kind":"key-pair","status":"observed-clean-at-scan-time"}],
+		"items":[{"resourceId":"pg-1","kind":"placement-group","disposition":"blocked","reason":"instance-members","memberCount":1}],
+		"counts":[{"kind":"key-pair","class":"public-key-object-without-metadata","disposition":"deleted","count":2}]},
+		"accounting":[{"kind":"placement-group","status":"accounted","remaining":1}]}`)
+	goldenJSON(t, ExerciseResult{Skipped: ExerciseSkippedNoQuiescedKind}, `{"skipped":"no-quiesced-kind"}`)
 }
 
 func TestAdminWire_ListAccountsResponse(t *testing.T) {

contracts/viperblockd/legacy/v1/README.md

@@ -51,19 +51,6 @@ request/reply replies.
 removed its local state and export. These routes and JSON shapes are deployed
 legacy compatibility behaviour, not a generic EBS-provider interface.
 
-## Fence route
-
-`VolumeFencedSubject(node)` is `ebs.<node>.fenced`, or `ebs.fenced` with no
-node for a single-node daemon. The Viperblock adapter publishes
-`VolumeFencedEvent` on it after tearing down an export whose volume lease
-moved to another node; the local daemon subscribes to stop the guest that
-was using it. Node-addressed, like mount and unmount: the guest is local, and
-no other node's daemon has anything to stop.
-
-The event is sent as a request and repeated until the daemon replies with `VolumeFencedAck`, because the fence often runs while the node's NATS server is restarting and a single publish can reach no subscriber.
-The daemon replies only when the event carries a reply subject, so a plain publish is still accepted.
-Repeats are harmless, since fencing a guest that is already fenced changes nothing.
-
 ## Abandon route
 
 `VolumeAbandonSubject(node)` is `ebs.<node>.abandon`. The daemon uses

contracts/viperblockd/legacy/v1/abandon_test.go

@@ -9,9 +9,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// TestVolumeAbandonSubject_BothForms pins the abandon route the same way the
-// fenced route is pinned: node-addressed, with an empty-node fallback for a
-// single-node daemon.
+// TestVolumeAbandonSubject_BothForms pins the abandon route: node-addressed,
+// with an empty-node fallback for a single-node daemon.
 func TestVolumeAbandonSubject_BothForms(t *testing.T) {
 	assert.Equal(t, "ebs.node-a.abandon", viperblocklegacyv1.VolumeAbandonSubject("node-a"))
 	assert.Equal(t, "ebs.abandon", viperblocklegacyv1.VolumeAbandonSubject(""))

contracts/viperblockd/legacy/v1/fence.go

@@ -1,29 +0,0 @@
-package viperblocklegacyv1
-
-// VolumeFencedSubject is where a fenced volume is announced so the daemon on
-// this node can stop the guest that was using it. Node-addressed: the guest is
-// local, and every other node's daemon has nothing to do about it.
-func VolumeFencedSubject(node string) string {
-	if node == "" {
-		return "ebs.fenced"
-	}
-	return "ebs." + node + ".fenced"
-}
-
-// VolumeFencedEvent announces that a node tore down a volume's export because
-// the lease moved. Winner is who holds it now, for the operator reading the
-// guest's stop reason.
-type VolumeFencedEvent struct {
-	Volume string `json:"volume"`
-	Node   string `json:"node"`
-	Winner string `json:"winner"`
-	Reason string `json:"reason"`
-}
-
-// VolumeFencedAck is the daemon's answer to a VolumeFencedEvent. Any answer
-// ends the sender's retries; Instance names the guest stopped, if one was, and
-// Error says why the daemon could not act on the event.
-type VolumeFencedAck struct {
-	Instance string `json:"instance,omitempty"`
-	Error    string `json:"error,omitempty"`
-}

contracts/viperblockd/legacy/v1/fence_test.go

@@ -1,54 +0,0 @@
-package viperblocklegacyv1_test
-
-import (
-	"encoding/json"
-	"testing"
-
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestVolumeFencedSubject_IsNodeAddressed pins the routing. A fenced guest is on
-// the node that lost the volume, so a subject without the node in it would reach
-// daemons with nothing to stop and, on a queue group, miss the one that has.
-func TestVolumeFencedSubject_IsNodeAddressed(t *testing.T) {
-	assert.Equal(t, "ebs.node-a.fenced", viperblocklegacyv1.VolumeFencedSubject("node-a"))
-	assert.Equal(t, "ebs.fenced", viperblocklegacyv1.VolumeFencedSubject(""),
-		"a single-node daemon has no node name, and still has to hear its own fences")
-}
-
-// TestVolumeFencedEvent_JSON pins the exact field spelling of the fenced
-// event, published by viperblockd and consumed by the daemon.
-func TestVolumeFencedEvent_JSON(t *testing.T) {
-	golden := `{"volume":"vol-1","node":"node-a","winner":"node-b","reason":"volume lease moved to node-b while this node had the volume open"}`
-	event := viperblocklegacyv1.VolumeFencedEvent{
-		Volume: "vol-1",
-		Node:   "node-a",
-		Winner: "node-b",
-		Reason: "volume lease moved to node-b while this node had the volume open",
-	}
-
-	marshaled, err := json.Marshal(event)
-	require.NoError(t, err)
-	assert.JSONEq(t, golden, string(marshaled))
-
-	var decoded viperblocklegacyv1.VolumeFencedEvent
-	require.NoError(t, json.Unmarshal([]byte(golden), &decoded))
-	assert.Equal(t, event, decoded)
-}
-
-// TestVolumeFencedAck_JSON pins the daemon's answer, which is what ends
-// viperblockd's retries of a fence notice.
-func TestVolumeFencedAck_JSON(t *testing.T) {
-	golden := `{"instance":"i-1","error":"bad event"}`
-	ack := viperblocklegacyv1.VolumeFencedAck{Instance: "i-1", Error: "bad event"}
-
-	marshaled, err := json.Marshal(ack)
-	require.NoError(t, err)
-	assert.JSONEq(t, golden, string(marshaled))
-
-	empty, err := json.Marshal(viperblocklegacyv1.VolumeFencedAck{})
-	require.NoError(t, err)
-	assert.JSONEq(t, `{}`, string(empty), "an ack for a volume with no local guest carries nothing")
-}

docs/DESIGN.md

@@ -65,9 +65,10 @@ The EC2 registration (`spinifex/domains/ec2/awsapi`, `NewRegistration`) parses t
 The gateway communicates with daemons via NATS request/response. Most calls go through `natsmsg.NATSRequest`, which marshals the input, attaches the account ID as a NATS header, and unmarshals the typed response:
 
 ```go
-func (s *NATSInstanceService) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
-    topic := fmt.Sprintf("ec2.RunInstances.%s", aws.StringValue(input.InstanceType))
-    return natsmsg.NATSRequest[ec2.Reservation](ctx, s.natsConn, topic, input, 5*time.Minute, accountID)
+func runIntoReservation(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, accountID, crID string) (*ec2.Reservation, error) {
+    subject := "ec2.RunInstances.cr." + crID
+    reservation, err := natsmsg.NATSRequest[ec2.Reservation](ctx, natsConn, subject, input, 5*time.Minute, accountID)
+    // ...
 }
 ```
 
@@ -111,14 +112,14 @@ When a daemon handles `RunInstances`:
 
 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
 2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
-3. **Volume Mount**: Sends a mount request to the colocated Viperblock worker on `ebs.{node}.mount`, receives NBD URI
+3. **Volume Mount**: Publishes each volume through the EBS provider contract (`PublishVolume` on `ebs.provider.v1.{node}.mount`), receives its export
 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management
 6. **Command subscription**: Subscribes to `ec2.cmd.{instanceID}` for subsequent commands on this VM
 
 ```go
-// Volume mounting via NATS (runtime/daemon/vm_adapters.go)
-reply, err := ebsRequestWithTrace(ctx, a.nc, instance.AccountID, viperblocklegacyv1.MountSubject(a.node), ebsMountRequest, 30*time.Second)
+// Volume mounting through the EBS provider contract (runtime/daemon/vm_adapters.go)
+published, err := a.provider.PublishVolume(ebsprovider.WithAccountID(ctx, accountID), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: req.Name, NodeID: a.node})
 
 // QEMU launch with NBD-backed drives in args (runtime/compute/vm/vm.go)
 cmd := exec.Command(qemuArchitecture, args...) // qemu-system-x86_64 or qemu-system-aarch64

docs/compute/launching-instances/README.md

@@ -23,6 +23,7 @@ resources:
 - [Overview](#overview)
 - [Launch](#launch)
 - [Manage](#manage)
+- [Stopping and Data Volumes](#stopping-and-data-volumes)
 - [Modify Instance Attributes](#modify-instance-attributes)
 - [Instance Metadata Options](#instance-metadata-options)
 - [Spot Instances](#spot-instances)
@@ -73,6 +74,31 @@ aws ec2 terminate-instances --instance-ids $INSTANCE_ID
 aws ec2 reboot-instances --instance-ids $INSTANCE_ID
 ```
 
+## Stopping and Data Volumes
+
+A guest that powers itself off (`poweroff`, `shutdown -h`) goes `stopping` then `stopped`, the same as `stop-instances`.
+It is not restarted automatically.
+`InstanceInitiatedShutdownBehavior=terminate` is not honoured yet: a guest shutdown always stops the instance.
+
+Data volumes stay `in-use` and attached to a stopped instance, and `start-instances` mounts them again.
+While the instance is stopped, `attach-volume` of one of its data volumes to another instance returns `VolumeInUse`, and `delete-volume` is refused.
+
+`detach-volume` works on a stopped instance, with or without `--instance-id`.
+The volume becomes `available` and the next start does not attach it.
+The root volume of a stopped instance can be detached too, as on AWS.
+The internal EFI variable store is not a customer volume: it never appears in `describe-volumes` or `BlockDeviceMappings`, and detaching it is refused with `OperationNotPermitted`.
+The instance then cannot start: `start-instances` returns `InvalidParameterValue` with "Instance does not have a volume attached at root (/dev/sda1)".
+
+`attach-volume` works on a stopped instance too.
+The volume must be `available` and in the instance's availability zone, otherwise the call returns `VolumeInUse` or `InvalidVolume.ZoneMismatch`.
+The attachment is recorded on the stopped instance and the volume becomes `in-use` straight away.
+Attaching at the root device name (for example `/dev/sda1`) makes the volume the root again, and the next `start-instances` boots from it.
+
+An instance stopped by an earlier build had its data volumes released to `available` while its record still listed them.
+On start, a listed data volume that was deleted or is now attached to another instance is dropped with a warning in the daemon log, and the start continues.
+A listed volume that is still `available` and unattached is attached again.
+A missing root volume fails the start with the same `InvalidParameterValue` answer.
+
 ## Modify Instance Attributes
 
 Change instance type, user data, or termination protection. Instance type and user data require the instance to be **stopped** first.
@@ -135,6 +161,30 @@ aws ec2 get-console-output --instance-id $INSTANCE_ID \
   --query 'Output' --output text | base64 -d
 ```
 
+Like AWS, the call returns the most recent 64 KiB of the console.
+While an instance is shutting down after a terminate request, the call reads the renamed file described below.
+Once the instance has left the node's running set, which includes a stopped or terminated instance, the node no longer answers and the call returns `InvalidInstanceID.NotFound`.
+AWS still lists a terminated instance for a while after termination, so a caller may expect this call to work then; it does not here, a known gap.
+Whether the API should answer for a stopped instance is not settled here; only the file on the host is kept.
+
+### Where console logs live and how big they get
+
+The serial console of each instance is written on its host to `/run/spinifex/console-<instance-id>.log`.
+`/run` is memory-backed, so these files are cleared when the host reboots, including any left by older releases.
+The daemon keeps them bounded.
+A file over 4 MiB is cut back in place to its newest 1 MiB, and the guest keeps writing while it happens.
+The console is opened in append mode, so a reboot or auto-restart of the guest adds to the same file instead of starting over.
+A live `console-<instance-id>.log` is deleted only when it has not changed for an hour and no QEMU process on the host has it open, so a stopped instance keeps its console for an hour and a quiet running guest keeps it indefinitely.
+If the daemon cannot read the open files of every QEMU process on the host, it deletes nothing on that pass and logs a warning.
+This also clears the file of an instance that was stopped and then terminated, or that was recovered on another node, within about an hour of its last write.
+When an instance is terminated, including a failed first launch and the system instances behind load balancers, queues and EKS control planes, its file is renamed to `console-<instance-id>.log.terminated`.
+The daemon removes a `.terminated` file an hour after its last write.
+
+Known limits:
+
+- The log shipper follows only `console-*.log`, so it stops following a file when it is renamed on terminate, and the last few seconds of output that it had not yet read can be lost.
+- The instance password retrieval reads only the live file name, so it does not find a console renamed after terminate.
+
 ## Instance Types
 
 List instance types available on the current host. The catalog is generated from the host CPU (Intel, AMD, or ARM) and includes burstable (t-family), general purpose (m-family), compute optimised (c-family), and memory optimised (r-family) types.

docs/containers/eks/README.md

@@ -59,6 +59,9 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM
 
 - **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
 - **The cluster security group is auto-managed.** As on EKS, Spinifex creates one `eks-cluster-sg-<name>-<id>` in your VPC, tagged `kubernetes.io/cluster/<name>=owned` and `aws:eks:cluster-name`, allowing all traffic from itself. It is attached to the workers and the private-endpoint network interface, and `DescribeCluster` reports it as `clusterSecurityGroupId`. A launch template that names security groups replaces it on the workers, as on AWS. Clusters created before this change keep their `-control-plane-sg` and `-nodegroup-sg` until you recreate them. Deleting a cluster deletes the cluster security group and removes only the rules on your other security groups that reference it. `vpc_config.security_group_ids` is ignored.
+- **Node group deletes are asynchronous.** `DeleteNodegroup` returns at once with the group `DELETING`, and `DescribeNodegroup` answers `ResourceNotFoundException` once the workers and the group's instance profile are gone. Deleting a cluster or creating a group of the same name while a group is `DELETING` returns `ResourceInUseException`, as does a second delete of a `DELETING` group.
+- **If a node group delete cannot finish,** the group moves to `DELETE_FAILED` after 30 minutes, with a `NODE_TERMINATION_FAILURE` health issue naming the workers that are still running. Run `DeleteNodegroup` again to resume it.
+- **Each node group owns an EKS-managed instance profile.** Spinifex creates it, tags it `spinifex:eks-cluster`, `spinifex:eks-nodegroup` and `spinifex:eks-profile-id`, and puts the node role in it. It is removed with the group, so you can delete the node role afterwards with no extra steps. Spinifex never deletes an instance profile it did not create, or one whose tags, name or role no longer match what it recorded; it leaves the profile and logs a warning with its name.
 - **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
 - **Workers launch with IMDSv2 required and an IMDS hop limit of 2.**
   This matches AWS managed node groups and lets a pod without IRSA reach the node role through the SDK default credential chain.
@@ -377,6 +380,15 @@ Control plane readiness and kubectl authentication above lists which clusters re
 
 **`create-cluster` rejected with an authentication-mode error.** `CONFIG_MAP` is the one mode Spinifex rejects, because it has no `aws-auth` ConfigMap path. Set `authentication_mode` to `API` or `API_AND_CONFIG_MAP`, or leave it unset, and grant access with access entries.
 
+**`delete-role` fails with `DeleteConflict` after deleting a node group.** Node groups created before per-group instance profiles left an instance profile named after the role, which Spinifex no longer removes because it cannot prove it created it. Remove it yourself:
+
+```bash
+aws iam remove-role-from-instance-profile --instance-profile-name <role-name> --role-name <role-name>
+aws iam delete-instance-profile --instance-profile-name <role-name>
+```
+
+**A node group is `DELETE_FAILED`.** `aws eks describe-nodegroup` lists the workers that did not stop under `health.issues`. Once they are gone, or after you fix the cause, run `aws eks delete-nodegroup` again.
+
 **Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, use the AWS form from `DescribeAddonVersions` (see Add-on versions above), or leave `addon_version` unset so Spinifex selects the catalog default.
 
 ## Reference Architectures

docs/package-boundary/eks-cluster-lifecycle.md

@@ -34,7 +34,7 @@ Node group workers and CSI-provisioned EBS volumes are never removed by a cluste
 
 ## Gaps
 
-1. Four `eks.*` handlers still hold their request for work that grows with the resource, and the subject table names them as not conformant: `CreateCluster` (reclaiming a `FAILED` cluster of the same name tears its infra down inline), `DeleteNodegroup` (one awaited terminate per worker), `UpdateNodegroupConfig` (one awaited launch or terminate per worker added or removed) and `RestoreSnapshot` (launches a control-plane VM and fences the old one inline).
+1. Three `eks.*` handlers still hold their request for work that grows with the resource, and the subject table names them as not conformant: `CreateCluster` (reclaiming a `FAILED` cluster of the same name tears its infra down inline), `UpdateNodegroupConfig` (one awaited launch or terminate per worker added or removed) and `RestoreSnapshot` (launches a control-plane VM and fences the old one inline).
 2. AWS's answer to a repeat delete on a `DELETING` cluster has not been observed; `DELETING` from the record is the chosen behaviour.
 3. Control-plane, load balancer and VPC resources in the system account are keyed by the bare cluster name, so a teardown by name can reach another tenant's same-named cluster.
 4. The teardown deletes load balancers the in-cluster controller created in the customer account, which AWS leaves behind.

docs/vultr-integration/README.md

@@ -265,7 +265,12 @@ If this returns a list of available instance types, your installation is working
 
 Spinifex is running, but it holds nothing yet: no machine images, no networks, no instances.
 
-Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance.
+Log in to node 1, and follow [Setting Up Your Cluster](/docs/setting-up-your-cluster) there to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance:
+
+```bash
+./deploy-vultr.sh ssh
+```
+
 An instance you launch into a public subnet gets a Vultr reserved IP, which you can see in the Vultr portal under **Network → Reserved IPs**.
 
 ## 8. Test resiliency

internal/testkit/drainsocket.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"sync/atomic"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -28,10 +29,11 @@ func SocketTempDir(t *testing.T) string {
 // StartDrainSocket stands in for the NBD plugin's drain socket, at the path the
 // plugin serves it on the node hosting volumeID ({dataDir}/viperblock/{volumeID}
 // /snapshot.sock). Every connection is answered with ack ("OK\n" for a drain
-// that reached S3, "ERR\n" for one that did not) and closed.
-func StartDrainSocket(t *testing.T, dataDir, volumeID, ack string) {
+// that reached S3, "ERR\n" for one that did not) and closed. connections
+// reports how many drains have been asked for so far.
+func StartDrainSocket(t *testing.T, dataDir, volumeID, ack string) (connections func() int) {
 	t.Helper()
-	serveDrainSocket(t, dataDir, volumeID, ack, nil, nil)
+	return serveDrainSocket(t, dataDir, volumeID, ack, nil, nil)
 }
 
 // StartBlockingDrainSocket stands in for a plugin whose flush is still running:
@@ -43,7 +45,7 @@ func StartBlockingDrainSocket(t *testing.T, dataDir, volumeID, ack string) (acce
 
 	acceptedCh := make(chan struct{})
 	releaseCh := make(chan struct{})
-	serveDrainSocket(t, dataDir, volumeID, ack, acceptedCh, releaseCh)
+	_ = serveDrainSocket(t, dataDir, volumeID, ack, acceptedCh, releaseCh)
 
 	var once sync.Once
 	t.Cleanup(func() { once.Do(func() { close(releaseCh) }) })
@@ -52,8 +54,9 @@ func StartBlockingDrainSocket(t *testing.T, dataDir, volumeID, ack string) (acce
 }
 
 // serveDrainSocket answers every connection with ack, optionally closing
-// accepted on the first one and waiting for release before it replies.
-func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan struct{}, release <-chan struct{}) {
+// accepted on the first one and waiting for release before it replies. The
+// returned func counts the connections accepted so far.
+func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan struct{}, release <-chan struct{}) func() int {
 	t.Helper()
 
 	dir := filepath.Join(dataDir, "viperblock", volumeID)
@@ -63,6 +66,7 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 	require.NoError(t, err)
 	t.Cleanup(func() { _ = ln.Close() })
 
+	var count atomic.Int64
 	go func() {
 		var once sync.Once
 		for {
@@ -70,6 +74,7 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 			if err != nil {
 				return // listener closed by the cleanup above
 			}
+			count.Add(1)
 			if accepted != nil {
 				once.Do(func() { close(accepted) })
 			}
@@ -80,4 +85,5 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 			_ = conn.Close()
 		}
 	}()
+	return func() int { return int(count.Load()) }
 }

scripts/images/eks-node/eks-node-role.sh

@@ -159,6 +159,7 @@ case "${ROLE}" in
         svc_start mulga-eks-state-report
         svc_start mulga-eks-addon-sync
         svc_start mulga-eks-node-gc
+        svc_optional mulga-eks-log-forward
         svc_optional eks-pod-identity-webhook
         ;;
     server-join)
@@ -178,12 +179,14 @@ case "${ROLE}" in
         svc_start konnectivity-server
         svc_start mulga-eks-state-report
         svc_start mulga-eks-node-gc
+        svc_optional mulga-eks-log-forward
         svc_optional eks-pod-identity-webhook
         ;;
     agent)
         log "configuring agent role"
         svc_enable k3s-agent
         svc_start k3s-agent
+        svc_optional mulga-eks-log-forward
         ;;
     *)
         die "SPINIFEX_K3S_ROLE missing or invalid: '${ROLE}'"

scripts/images/eks-node/eks-node-role_test.sh

@@ -146,6 +146,15 @@ for INIT in openrc systemd; do
     )"
     check "server(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
     check "server(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
+    check "server(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server(${INIT}): starts log-forward before the webhook" "yes" "$(
+        awk -v f="$(start_cmd "${INIT}" mulga-eks-log-forward)" -v w="$(start_cmd "${INIT}" eks-pod-identity-webhook)" '
+            index($0, f) == 1 { fn = NR }
+            index($0, w) == 1 { wn = NR }
+            END { print (fn > 0 && wn > fn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
     check "server(${INIT}): no k3s-agent" "no" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
     check "server(${INIT}): pod-identity-webhook is the last call" "$(start_cmd "${INIT}" eks-pod-identity-webhook)" "$(grep -v "$(disable_cmd "${INIT}" eks-node-role)" "${CALLS}" | tail -n 1)"
@@ -196,6 +205,15 @@ for INIT in openrc systemd; do
     # published the cluster-identical artifacts, and a join re-publish would
     # only race the bootstrap bus. This is deliberate, not an oversight.
     check "server-join(${INIT}): pod-identity-webhook is the last call" "$(start_cmd "${INIT}" eks-pod-identity-webhook)" "$(grep -v "$(disable_cmd "${INIT}" eks-node-role)" "${CALLS}" | tail -n 1)"
+    check "server-join(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server-join(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server-join(${INIT}): starts log-forward before the webhook" "yes" "$(
+        awk -v f="$(start_cmd "${INIT}" mulga-eks-log-forward)" -v w="$(start_cmd "${INIT}" eks-pod-identity-webhook)" '
+            index($0, f) == 1 { fn = NR }
+            index($0, w) == 1 { wn = NR }
+            END { print (fn > 0 && wn > fn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
     check "server-join(${INIT}): no k3s-first-boot" "yes" "$(absent_all "${CALLS}" "k3s-first-boot")"
     check "server-join(${INIT}): no k3s-agent" "yes" "$(absent_all "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server-join(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
@@ -207,8 +225,39 @@ for INIT in openrc systemd; do
     check "agent(${INIT}): exit 0" 0 "${rc}"
     check "agent(${INIT}): role file" "agent" "$(cat "${ROLE_FILE}")"
     check "agent(${INIT}): enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
+    check "agent(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "agent(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "agent(${INIT}): log-forward is enabled after k3s-agent" "yes" "$(
+        awk -v k="$(enable_cmd "${INIT}" k3s-agent)" -v f="$(enable_cmd "${INIT}" mulga-eks-log-forward)" '
+            index($0, k) == 1 { kn = NR }
+            index($0, f) == 1 { fn = NR }
+            END { print (kn > 0 && fn > kn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
+
     check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "eks-pod-identity-webhook" "k3s-first-boot" "mulga-eks-node-gc" "$(enable_cmd "${INIT}" k3s)")"
 
+    # A broken log forwarder must never stop a node joining.
+    for FAILING in enable start; do
+        run_case "${INIT}" agent_logfwd_${FAILING}_fail
+        printf 'SPINIFEX_K3S_ROLE=agent\n' > "${EKS_NODE_ENVFILE}"
+        if [ "${FAILING}" = enable ]; then
+            STUB_FAIL="$(enable_cmd "${INIT}" mulga-eks-log-forward)"
+            export STUB_FAIL
+            STUB_FAIL="${STUB_FAIL#rc-update add }"; STUB_FAIL="${STUB_FAIL#systemctl enable }"
+        else
+            STUB_FAIL="$(start_cmd "${INIT}" mulga-eks-log-forward)"
+            export STUB_FAIL
+            STUB_FAIL="${STUB_FAIL#rc-service }"; STUB_FAIL="${STUB_FAIL#systemctl start --no-block }"
+        fi
+        invoke
+        unset STUB_FAIL
+        check "agent(${INIT}): log-forward ${FAILING} failure still exits 0" 0 "${rc}"
+        check "agent(${INIT}): log-forward ${FAILING} failure still enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
+        check "agent(${INIT}): log-forward ${FAILING} failure still starts k3s-agent" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" k3s-agent)")"
+        check "agent(${INIT}): log-forward ${FAILING} failure still self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
+    done
+
     # --- agent (inferred from agent.env, no explicit role) ---
     run_case "${INIT}" agent_inferred
     printf 'K3S_URL=https://nlb:443\nK3S_TOKEN=abc\n' > "${EKS_NODE_AGENT_ENVFILE}"

scripts/images/eks-node/manifest.conf

@@ -45,6 +45,8 @@ scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-s
 scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \
 scripts/images/eks-node/mulga-eks-node-gc.initd:/etc/init.d/mulga-eks-node-gc \
 scripts/images/eks-node/mulga-eks-node-gc.sh:/usr/local/sbin/mulga-eks-node-gc \
+scripts/images/eks-node/mulga-eks-log-forward.initd:/etc/init.d/mulga-eks-log-forward \
+scripts/images/eks-node/mulga-eks-log-forward.sh:/usr/local/sbin/mulga-eks-log-forward \
 scripts/images/eks-node/konnectivity-server.initd:/etc/init.d/konnectivity-server \
 scripts/images/eks-node/konnectivity-server-prestart.sh:/usr/local/sbin/konnectivity-server-prestart \
 scripts/images/eks-node/konnectivity/konnectivity-agent.yaml:/usr/share/spinifex-eks/konnectivity/konnectivity-agent.yaml \

scripts/images/eks-node/mulga-eks-log-forward.initd

@@ -0,0 +1,30 @@
+#!/sbin/openrc-run
+# mulga-eks-log-forward — copies k3s warnings and errors to the serial console
+# so they reach the Elasticsearch sink. Best effort: nothing depends on it and
+# a failure here never affects the node.
+
+description="Forward k3s warnings and errors to the serial console"
+command="/bin/sh"
+command_background="yes"
+supervisor="supervise-daemon"
+respawn_delay=5
+pidfile="/run/mulga-eks-log-forward.pid"
+output_log="/dev/null"
+error_log="/dev/null"
+
+depend() {
+    after k3s k3s-agent
+}
+
+start_pre() {
+    role=$(cat /etc/spinifex-eks/role 2>/dev/null || true)
+    if [ "${role}" = "agent" ]; then
+        log=/var/log/k3s-agent.log
+        FORWARD_PREFIX="[k3s-agent]"
+    else
+        log=/var/log/k3s.log
+        FORWARD_PREFIX="[k3s]"
+    fi
+    export FORWARD_PREFIX
+    command_args="-c 'tail -n +1 -F ${log} 2>/dev/null | /usr/local/sbin/mulga-eks-log-forward'"
+}

scripts/images/eks-node/mulga-eks-log-forward.sh

@@ -0,0 +1,90 @@
+#!/bin/sh
+# mulga-eks-log-forward — copy the warnings and errors of a k3s log to the
+# serial console, where the host picks them up for the Elasticsearch sink.
+#
+# Reads the tailed log on stdin. Lines that matter are redacted first, then
+# cut to FORWARD_MAX_BYTES, then rate limited: at most FORWARD_RATE_PER_MIN a
+# minute reach the console and the rest are reported as one "N lines dropped"
+# line. Every console write has a timeout, so a stuck console loses lines
+# instead of stalling the tail or k3s.
+#
+# Env: FORWARD_CONSOLE (default /dev/console), FORWARD_PREFIX, FORWARD_MAX_BYTES
+# (512), FORWARD_RATE_PER_MIN (60), FORWARD_WRITE_TIMEOUT seconds (2) and
+# FORWARD_CLOCK_CMD (default: seconds of uptime, read without forking).
+
+CONSOLE=${FORWARD_CONSOLE:-/dev/console}
+PREFIX=${FORWARD_PREFIX:-[k3s]}
+MAX_BYTES=${FORWARD_MAX_BYTES:-512}
+RATE=${FORWARD_RATE_PER_MIN:-60}
+WRITE_TIMEOUT=${FORWARD_WRITE_TIMEOUT:-2}
+CLOCK_CMD=${FORWARD_CLOCK_CMD:-}
+
+now() {
+    if [ -n "${CLOCK_CMD}" ]; then
+        sh -c "${CLOCK_CMD}"
+    else
+        read -r up _ < /proc/uptime && echo "${up%%.*}"
+    fi
+}
+
+# emit TEXT: one console line, abandoned after WRITE_TIMEOUT. Returns non-zero
+# when the console did not take it.
+emit() {
+    timeout "${WRITE_TIMEOUT}" sh -c 'printf "%s\n" "$1" >> "$2"' sh "$1" "${CONSOLE}" 2>/dev/null
+}
+
+# Filter, redact and truncate in one awk pass so lines that do not matter cost
+# no fork. Redaction runs before the cut so a secret straddling the limit
+# cannot leave a readable prefix behind.
+select_lines() {
+    LC_ALL=C awk -v max="${MAX_BYTES}" '
+    {
+        line = $0
+        gsub(/\r/, "", line)
+        keep = 0
+        if (line ~ /level=(warning|error)/) keep = 1
+        else if (line ~ /^[WE][0-9][0-9][0-9][0-9] /) keep = 1
+        else if (line ~ /NodeNotReady|PLEG|[Ll]ease|[Nn]ode status/) keep = 1
+        if (!keep) next
+
+        gsub(/[Bb]earer +[A-Za-z0-9._~+\/=-]+/, "Bearer [REDACTED]", line)
+        gsub(/K10[0-9a-f]+(::[A-Za-z0-9-]+:[^ "\047]+)?/, "[REDACTED]", line)
+        gsub(/eyJ[A-Za-z0-9_.-]+/, "[REDACTED]", line)
+        gsub(/(AKIA|ASIA)[A-Z0-9]+/, "[REDACTED]", line)
+        gsub(/X-Amz-Signature=[^& "\047]*/, "X-Amz-Signature=[REDACTED]", line)
+        gsub(/X-Amz-Credential=[^& "\047]*/, "X-Amz-Credential=[REDACTED]", line)
+        gsub(/X-Amz-Security-Token=[^& "\047]*/, "X-Amz-Security-Token=[REDACTED]", line)
+        gsub(/[^[:print:]\t]/, " ", line)
+        print substr(line, 1, max)
+    }'
+}
+
+win_start=0
+sent=0
+dropped=0
+
+flush_dropped() {
+    [ "${dropped}" -gt 0 ] || return 0
+    emit "${PREFIX} ${dropped} lines dropped"
+    dropped=0
+}
+
+select_lines | {
+    while IFS= read -r line; do
+        t=$(now) || t=0
+        if [ "${win_start}" -eq 0 ] || [ $((t - win_start)) -ge 60 ]; then
+            flush_dropped
+            win_start=${t}
+            sent=0
+            [ "${win_start}" -eq 0 ] && win_start=1
+        fi
+        if [ "${sent}" -ge "${RATE}" ]; then
+            dropped=$((dropped + 1))
+            continue
+        fi
+        sent=$((sent + 1))
+        emit "${PREFIX} ${line}" || dropped=$((dropped + 1))
+    done
+    flush_dropped
+}
+exit 0

scripts/images/eks-node/mulga-eks-log-forward_test.sh

@@ -0,0 +1,190 @@
+#!/bin/sh
+# Self-contained POSIX test for mulga-eks-log-forward.sh: the line filter,
+# redaction, truncation, rate cap and blocked-console behaviour. The script
+# reads the tailed log on stdin and writes to FORWARD_CONSOLE, which the test
+# points at a temp file (or a reader-less FIFO), so no root or tty is needed.
+#
+# Run: sh scripts/images/eks-node/mulga-eks-log-forward_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+SCRIPT="${SCRIPT_DIR}/mulga-eks-log-forward.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+PASS=0
+FAIL=0
+check() { # check <desc> <expected> <actual>
+    if [ "$2" = "$3" ]; then
+        PASS=$((PASS + 1))
+    else
+        FAIL=$((FAIL + 1))
+        echo "FAIL: $1"
+        echo "  expected: [$2]"
+        echo "  actual:   [$3]"
+    fi
+}
+
+# forward <name>: feeds stdin to the script with a per-case console file and a
+# fixed clock, leaving the forwarded text in ${OUT}.
+forward() {
+    CASE="${WORK}/$1"
+    mkdir -p "${CASE}"
+    OUT="${CASE}/console"
+    : > "${OUT}"
+    FORWARD_CONSOLE="${OUT}" FORWARD_PREFIX="[k3s-agent]" \
+        FORWARD_CLOCK_CMD="echo 1000" sh "${SCRIPT}" 2>/dev/null || true
+}
+forwarded() { grep -c . "${OUT}" || true; }
+
+# --- filter: lines that must pass ---
+forward filter_pass <<'EOF'
+time="2026-11-03T03:00:00Z" level=warning msg="slow response from apiserver"
+time="2026-11-03T03:00:01Z" level=error msg="failed to renew"
+W1103 03:00:02.123456    1234 reflector.go:561] watch of *v1.Pod ended
+E1103 03:00:03.123456    1234 kubelet.go:2400] node not found
+I1103 03:00:04.123456    1234 kubelet_node_status.go:73] Node status update failed
+I1103 03:00:05.123456    1234 controller.go:193] Failed to update lease
+I1103 03:00:06.123456    1234 kubelet.go:2200] PLEG is not healthy
+time="2026-11-03T03:00:07Z" level=info msg="Event: NodeNotReady kubelet"
+EOF
+check "filter: all eight interesting lines pass" "8" "$(forwarded)"
+check "filter: lines carry the prefix" "8" "$(grep -c '^\[k3s-agent\] ' "${OUT}" || true)"
+
+# --- filter: lines that must be dropped ---
+forward filter_drop <<'EOF'
+time="2026-11-03T03:00:00Z" level=info msg="Starting k3s agent"
+time="2026-11-03T03:00:01Z" level=debug msg="tick"
+I1103 03:00:02.123456    1234 server.go:100] Version: v1.32.5
+I1103 03:00:03.123456    1234 kubelet.go:1000] SyncLoop (UPDATE, "api")
+EOF
+check "filter: info and debug lines are dropped" "0" "$(forwarded)"
+
+# --- redaction: one case per secret shape ---
+redact() { # redact <name> <line> <secret-that-must-not-appear>
+    forward "redact_$1" <<EOF
+time="2026-11-03T03:00:00Z" level=error msg="$2"
+EOF
+    check "redact(${1}): line still forwarded" "1" "$(forwarded)"
+    if grep -qF "$3" "${OUT}"; then
+        check "redact(${1}): secret removed" "absent" "present"
+    else
+        check "redact(${1}): secret removed" "absent" "absent"
+    fi
+    check "redact(${1}): marker present" "1" "$(grep -c 'REDACTED' "${OUT}" || true)"
+}
+redact bearer "request failed Authorization: Bearer abc123SECRETvalue456" "abc123SECRETvalue456"
+redact k3s_token "join failed token K1009f3c2d4e5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2::server:s3cr3tPassw0rd" "s3cr3tPassw0rd"
+redact jwt "bad token eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.c2lnbmF0dXJlLXNlY3JldA" "eyJhbGciOiJIUzI1NiJ9"
+redact akia "credentials AKIAIOSFODNN7EXAMPLE rejected" "AKIAIOSFODNN7EXAMPLE"
+redact asia "credentials ASIAIOSFODNN7EXAMPLE rejected" "ASIAIOSFODNN7EXAMPLE"
+redact amz_signature "GET /obj?X-Amz-Signature=0123456789abcdef0123456789abcdef&X-Amz-Expires=60 failed" "0123456789abcdef0123456789abcdef"
+redact amz_credential "GET /obj?X-Amz-Credential=ZZZsecretscope%2F20261103%2Fap%2Fs3%2Faws4_request failed" "ZZZsecretscope"
+
+# --- truncation: text over 512 bytes is cut ---
+LONG=$(awk 'BEGIN { for (i = 0; i < 2000; i++) printf "a"; }')
+forward truncate <<EOF
+time="2026-11-03T03:00:00Z" level=error msg="${LONG}"
+EOF
+check "truncate: still forwarded" "1" "$(forwarded)"
+BODY=$(sed 's/^\[k3s-agent\] //' "${OUT}" | head -n 1)
+check "truncate: body is exactly 512 bytes" "512" "$(printf '%s' "${BODY}" | wc -c | tr -d ' ')"
+check "truncate: keeps the start of the line" "time=\"2026-11-03T03:00:00Z\"" "$(printf '%s' "${BODY}" | cut -c1-27)"
+
+# --- redaction runs before the cut: a secret over the limit leaves no prefix ---
+# The pad puts the 512-byte cut right after the bare marker (AKIA, eyJ, K10).
+# A cut fragment that long matches no redaction pattern, so redacting after the
+# cut would leave it in the output.
+HEAD='time="2026-11-03T03:00:00Z" level=error msg="'
+straddle() { # straddle <name> <secret> <bytes of the secret kept by the cut>
+    PAD=$(awk -v n=$((512 - ${#HEAD} - 1 - $3)) 'BEGIN { for (i = 0; i < n; i++) printf "x"; }')
+    forward "straddle_$1" <<EOF
+${HEAD}${PAD} $2 tail"
+EOF
+    check "straddle(${1}): line forwarded" "1" "$(forwarded)"
+    check "straddle(${1}): cut keeps the marker length" "$3" "$(sed 's/^\[k3s-agent\] //' "${OUT}" | head -n 1 | awk -v p="${PAD} " '{ print length($0) - index($0, p) - length(p) + 1 }')"
+    if grep -qF "$(printf '%s' "$2" | cut -c1-"$3")" "${OUT}"; then
+        check "straddle(${1}): no fragment survives the cut" "absent" "present"
+    else
+        check "straddle(${1}): no fragment survives the cut" "absent" "absent"
+    fi
+}
+straddle jwt "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.c2ln" 3
+straddle akia "AKIAIOSFODNN7EXAMPLE" 4
+straddle k3s_token "K1009f3c2d4e5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2::server:s3cr3tPassw0rd" 3
+
+# --- rate cap: 60 a minute, then one summary of what was dropped ---
+gen() { # gen <count>
+    awk -v n="$1" 'BEGIN { for (i = 0; i < n; i++) printf "time=\"t\" level=error msg=\"loop %d\"\n", i }'
+}
+gen 100 > "${WORK}/burst"
+forward rate_cap < "${WORK}/burst"
+check "rate cap: 60 lines forwarded plus one summary" "61" "$(forwarded)"
+check "rate cap: one dropped summary" "1" "$(grep -c '40 lines dropped' "${OUT}" || true)"
+check "rate cap: first line kept" "1" "$(grep -c 'loop 0"' "${OUT}" || true)"
+check "rate cap: line past the cap is not forwarded" "0" "$(grep -c 'loop 60"' "${OUT}" || true)"
+
+# --- rate cap: a new minute starts a new window and reports the old drops ---
+# The clock command advances 60 s every 70 calls, so lines 0-69 share a window
+# (10 dropped) and lines 70-74 land in the next one.
+CASE="${WORK}/rate_window"
+mkdir -p "${CASE}"
+OUT="${CASE}/console"
+: > "${OUT}"
+echo 0 > "${CASE}/calls"
+cat > "${CASE}/clock" <<EOF
+#!/bin/sh
+n=\$(cat "${CASE}/calls")
+echo \$((n + 1)) > "${CASE}/calls"
+echo \$((1000 + n / 70 * 60))
+EOF
+chmod +x "${CASE}/clock"
+gen 75 | FORWARD_CONSOLE="${OUT}" FORWARD_PREFIX="[k3s-agent]" FORWARD_CLOCK_CMD="${CASE}/clock" sh "${SCRIPT}" 2>/dev/null || true
+check "window: 60 + 5 forwarded plus one summary" "66" "$(forwarded)"
+check "window: summary names the 10 dropped" "1" "$(grep -c '10 lines dropped' "${OUT}" || true)"
+check "window: new window forwards its lines" "1" "$(grep -c 'loop 74"' "${OUT}" || true)"
+
+# --- blocked console: lines are dropped, the forwarder neither stalls nor fails ---
+CASE="${WORK}/blocked"
+mkdir -p "${CASE}"
+mkfifo "${CASE}/console"
+gen 3 > "${CASE}/in"
+(
+    FORWARD_CONSOLE="${CASE}/console" FORWARD_PREFIX="[k3s-agent]" FORWARD_CLOCK_CMD="echo 1000" \
+        FORWARD_WRITE_TIMEOUT=1 sh "${SCRIPT}" < "${CASE}/in"
+) &
+PID=$!
+i=0
+while kill -0 "${PID}" 2>/dev/null && [ "${i}" -lt 20 ]; do
+    sleep 1
+    i=$((i + 1))
+done
+if kill -0 "${PID}" 2>/dev/null; then
+    kill "${PID}" 2>/dev/null || true
+    check "blocked console: forwarder finished" "finished" "stalled"
+else
+    check "blocked console: forwarder finished" "finished" "finished"
+fi
+rc=0
+wait "${PID}" || rc=$?
+check "blocked console: exit 0" "0" "${rc}"
+
+# --- the service is wired in, and nothing depends on it ---
+INITD="${SCRIPT_DIR}/mulga-eks-log-forward.initd"
+check "initd ships" "yes" "$([ -f "${INITD}" ] && echo yes || echo no)"
+check "manifest installs the initd" "1" "$(grep -c 'mulga-eks-log-forward.initd:/etc/init.d/mulga-eks-log-forward' "${SCRIPT_DIR}/manifest.conf" || true)"
+check "manifest installs the script" "1" "$(grep -c 'mulga-eks-log-forward.sh:/usr/local/sbin/mulga-eks-log-forward' "${SCRIPT_DIR}/manifest.conf" || true)"
+DEPS=0
+for f in "${SCRIPT_DIR}"/*.initd; do
+    [ "${f}" = "${INITD}" ] && continue
+    if grep -E '^[[:space:]]*(need|use|want)[[:space:]].*mulga-eks-log-forward' "${f}" >/dev/null 2>&1; then
+        DEPS=$((DEPS + 1))
+    fi
+done
+check "nothing depends on the forwarder" "0" "${DEPS}"
+if [ -f "${INITD}" ]; then
+    check "initd does not need anything it could block boot on" "0" "$(grep -cE '^[[:space:]]*need[[:space:]]' "${INITD}" || true)"
+fi
+
+echo "mulga-eks-log-forward_test: ${PASS} passed, ${FAIL} failed"
+[ "${FAIL}" -eq 0 ]

scripts/images/eks-node/setup.sh

@@ -45,10 +45,10 @@ ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
 chmod 0755 /etc/init.d/eks-node-role /etc/init.d/k3s /etc/init.d/k3s-agent \
     /etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot /etc/init.d/mulga-eks-state-report \
     /etc/init.d/mulga-eks-addon-sync /etc/init.d/mulga-eks-node-gc /etc/init.d/konnectivity-server \
-    /etc/init.d/eks-pod-identity-webhook
+    /etc/init.d/eks-pod-identity-webhook /etc/init.d/mulga-eks-log-forward
 chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
     /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync \
-    /usr/local/sbin/mulga-eks-node-gc
+    /usr/local/sbin/mulga-eks-node-gc /usr/local/sbin/mulga-eks-log-forward
 # start_pre helpers shared with the systemd wrappers that will eventually
 # replace these runscripts.
 chmod 0755 /usr/local/sbin/k3s-prestart /usr/local/sbin/k3s-agent-prestart \

spinifex/agents/lambda/agent/bootstrap.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os/exec"
+	"path/filepath"
 	goruntime "runtime"
 	"syscall"
 	"time"
@@ -18,8 +19,8 @@ import (
 const sandboxID = 993
 
 // StartBootstrap starts the function's bootstrap at path as the sandbox uid and
-// gid with no supplementary groups, with no_new_privs, and with exactly env.
-// The caller waits on the returned command.
+// gid with no supplementary groups, with no_new_privs, and with exactly env,
+// in the bootstrap's own directory. The caller waits on the returned command.
 func StartBootstrap(ctx context.Context, path string, env []string, stdout, stderr io.Writer) (*exec.Cmd, error) {
 	cred := &syscall.Credential{Uid: sandboxID, Gid: sandboxID, Groups: []uint32{}}
 	return startBootstrap(ctx, path, env, stdout, stderr, cred)
@@ -30,6 +31,8 @@ func startBootstrap(ctx context.Context, path string, env []string, stdout, stde
 	// A nil Env inherits the agent's environment; the bootstrap gets only env.
 	cmd.Env = append([]string{}, env...)
 	cmd.Stdout, cmd.Stderr = stdout, stderr
+	// Packaged functions open their files relative to the task root.
+	cmd.Dir = filepath.Dir(path)
 	if cred != nil {
 		cmd.SysProcAttr = &syscall.SysProcAttr{Credential: cred}
 	}

spinifex/agents/lambda/agent/bootstrap_test.go

@@ -169,3 +169,11 @@ func TestStartBootstrapMissingExecutableFails(t *testing.T) {
 	_, err := startBootstrap(context.Background(), filepath.Join(t.TempDir(), "bootstrap"), nil, nil, nil, nil)
 	assert.Error(t, err)
 }
+
+func TestStartBootstrapRunsInItsOwnDirectory(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "data.txt"), []byte("packaged"), 0o644))
+	bootstrap := filepath.Join(dir, "bootstrap")
+	require.NoError(t, os.WriteFile(bootstrap, []byte("#!/bin/sh\ncat data.txt\n"), 0o755))
+	assert.Equal(t, "packaged", string(run(t, bootstrap, []string{"PATH=/bin:/usr/bin"}, false)))
+}

spinifex/bootstrap/config/config.go

@@ -5,6 +5,7 @@ package config
 import (
 	"fmt"
 	"log/slog"
+	"maps"
 	"net/netip"
 	"os"
 	"path/filepath"
@@ -355,6 +356,11 @@ type EBSConfig struct {
 	CacheSizeMB *int `json:"CacheSizeMB" mapstructure:"cache_size_mb"`
 }
 
+// SupportedEBSProvider reports whether name is a provider a node may run.
+func SupportedEBSProvider(name string) bool {
+	return name == EBSProviderViperblockd || name == EBSProviderQEMUNBD
+}
+
 // ResolvedProvider normalizes an empty Provider to EBSProviderViperblockd,
 // the default provider.
 func (c EBSConfig) ResolvedProvider() string {
@@ -801,6 +807,27 @@ func validateVultrPool(cc *ClusterConfig, p ExternalPool) error {
 	return nil
 }
 
+// validateSingleEBSProvider refuses nodes whose explicit [nodes.<name>.ebs]
+// providers differ. A peer stanza without one is unknown, not the default: the
+// generated config omits it, and the cluster's KV lock is the authoritative check.
+func validateSingleEBSProvider(cc *ClusterConfig) error {
+	var first, want string
+	for _, name := range slices.Sorted(maps.Keys(cc.Nodes)) {
+		if !viper.IsSet("nodes." + name + ".ebs.provider") {
+			continue
+		}
+		got := cc.Nodes[name].EBS.ResolvedProvider()
+		if first == "" {
+			first, want = name, got
+			continue
+		}
+		if got != want {
+			return fmt.Errorf("config: [nodes.%s.ebs] provider resolves to %q but [nodes.%s.ebs] resolves to %q; every node in a cluster must use the same EBS provider", first, want, name, got)
+		}
+	}
+	return nil
+}
+
 func validateClusterConfig(cc *ClusterConfig) error {
 	if viper.IsSet("network.external_dhcp") {
 		return fmt.Errorf("config: [network] external_dhcp is no longer supported; remove the key (static WAN-pool allocation only)")
@@ -809,9 +836,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		if viper.IsSet("nodes." + nodeName + ".vpcd.dhcp_bind_bridge") {
 			return fmt.Errorf("config: [nodes.%s.vpcd] dhcp_bind_bridge is no longer supported; remove the key (vpcd no longer runs a DHCP client)", nodeName)
 		}
-		switch nodeCfg.EBS.Provider {
-		case "", EBSProviderViperblockd, EBSProviderQEMUNBD:
-		case EBSProviderEmbedded:
+		switch {
+		case nodeCfg.EBS.Provider == "", SupportedEBSProvider(nodeCfg.EBS.Provider):
+		case nodeCfg.EBS.Provider == EBSProviderEmbedded:
 			// Refuse rather than silently upgrade: the embedded engine is gone,
 			// and starting anyway would migrate this node's volumes to
 			// ebsmetadata one-way without the operator ever asking for it.
@@ -829,6 +856,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 			return fmt.Errorf("config: [nodes.%s.ebs] cache_size_mb=%d must not be negative; use 0 to disable the cache, or remove the key for the default of %d", nodeName, *c, DefaultCacheSizeMB)
 		}
 	}
+	if err := validateSingleEBSProvider(cc); err != nil {
+		return err
+	}
 
 	if err := validateNorthstarCluster(cc); err != nil {
 		return err

spinifex/bootstrap/config/config_test.go

@@ -1108,6 +1108,45 @@ provider = "bogus"
 	assert.Contains(t, err.Error(), `provider="bogus"`)
 }
 
+func TestLoadConfig_EBSProvider_SingleProviderPerCluster(t *testing.T) {
+	stanza := func(node, provider string) string {
+		return fmt.Sprintf("[nodes.%s.ebs]\nprovider = %q\n", node, provider)
+	}
+	cases := []struct {
+		name    string
+		ebs     []string
+		wantErr string
+	}{
+		{"all unset", []string{"", "", ""}, ""},
+		{"explicit viperblockd beside unset peers", []string{"", stanza("n2", EBSProviderViperblockd), ""}, ""},
+		{"explicit qemunbd beside unset peers", []string{stanza("n1", EBSProviderQEMUNBD), "", ""}, ""},
+		{"all explicit qemunbd", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderQEMUNBD), stanza("n3", EBSProviderQEMUNBD)}, ""},
+		{"two explicit providers differ", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderViperblockd), ""},
+			`[nodes.n1.ebs] provider resolves to "qemunbd" but [nodes.n2.ebs] resolves to "viperblockd"`},
+		{"only the third node differs", []string{stanza("n1", EBSProviderViperblockd), stanza("n2", EBSProviderViperblockd), stanza("n3", EBSProviderQEMUNBD)},
+			`[nodes.n1.ebs] provider resolves to "viperblockd" but [nodes.n3.ebs] resolves to "qemunbd"`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			body := "node = \"n1\"\n"
+			for i, ebs := range tc.ebs {
+				body += fmt.Sprintf("\n[nodes.n%d]\nregion = \"us-east-1\"\n%s", i+1, ebs)
+			}
+			resetViper(t)
+			path := filepath.Join(t.TempDir(), "spinifex.toml")
+			require.NoError(t, os.WriteFile(path, []byte(body), 0600))
+
+			_, err := LoadConfig(path)
+			if tc.wantErr == "" {
+				require.NoError(t, err)
+				return
+			}
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.wantErr)
+		})
+	}
+}
+
 func TestEBSConfig_ResolvedTunablesDefault(t *testing.T) {
 	var c EBSConfig
 	assert.Equal(t, DefaultNBDKitThreads, c.ResolvedThreads())

spinifex/domains/acm/awsapi/registration.go

@@ -3,14 +3,11 @@ package awsapi
 import (
 	"context"
 	"errors"
-	"io"
 	"log/slog"
 	"maps"
 	"net/http"
 	"slices"
-	"strings"
 
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 	"github.com/nats-io/nats.go"
@@ -66,7 +63,7 @@ var Actions = map[string]Handler{
 func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
-		ResolveAction: resolveAction,
+		ResolveAction: dispatch.ResolveTargetAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeJSON,
 		Inventory:     OperationInventory(),
@@ -81,25 +78,11 @@ func OperationInventory() dispatch.Inventory {
 	}
 }
 
-// resolveAction is the one X-Amz-Target parser telemetry and dispatch share.
-func resolveAction(r *http.Request) string {
-	return actionFromTarget(r.Header.Get("X-Amz-Target"))
-}
-
-// actionFromTarget extracts the action suffix from an X-Amz-Target header.
-// Any "<Prefix>.<Action>" or bare "<Action>" form is accepted.
-func actionFromTarget(target string) string {
-	if _, after, ok := strings.CutLast(target, "."); ok {
-		return after
-	}
-	return target
-}
-
 // dispatch resolves, scopes and authorizes the request from one read of its
 // body, then runs the action. Errors are awserrors codes the gateway renders.
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
-	action := resolveAction(r)
+	action := dispatch.ResolveTargetAction(r)
 	if action == "" {
 		return errors.New(awserrors.ErrorMissingAction)
 	}
@@ -118,7 +101,7 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 		return errors.New(awserrors.ErrorInternalError)
 	}
 
-	body, err := readBoundedBody(r)
+	body, err := dispatch.ReadBoundedBody(r)
 	if err != nil {
 		slog.ErrorContext(r.Context(), "ACM dispatch: failed to read body", "err", err)
 		return err
@@ -144,16 +127,3 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	WriteJSONResponse(w, output)
 	return nil
 }
-
-// readBoundedBody reads the body under the SigV4 payload cap, so a body read
-// ahead of the policy gate can neither bypass it nor exhaust memory.
-func readBoundedBody(r *http.Request) ([]byte, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return body, nil
-}

spinifex/domains/acm/awsapi/registration_test.go

@@ -58,18 +58,6 @@ func dispatchWith(t *testing.T, deps Deps, inv dispatch.Invocation) (*httptest.R
 	return w, NewRegistration(deps).Dispatch(w, inv)
 }
 
-func TestActionFromTarget(t *testing.T) {
-	for target, want := range map[string]string{
-		"CertificateManager.ListCertificates": "ListCertificates",
-		"a.b.ListCertificates":                "ListCertificates",
-		"ListCertificates":                    "ListCertificates",
-		"CertificateManager.":                 "",
-		"":                                    "",
-	} {
-		assert.Equal(t, want, actionFromTarget(target), "target %q", target)
-	}
-}
-
 // TestActionsScopeTableIsExhaustive is what stops the next ACM action being
 // added with a silent account-wide grant. It asserts both directions, so a
 // scope left behind by a deleted or renamed action fails too.

spinifex/domains/ec2/accountcleanup/cleanup.go

@@ -49,11 +49,13 @@ type PlacementGroupOwner interface {
 	CleanupAccount(ctx context.Context, accountID string, referenced func(groupID, groupName string) string) (*ec2placementgroup.AccountCleanup, error)
 }
 
-// LaunchTemplateOwner is the launch-template owner's account scan and residue
-// cleanup; live templates are never deleted by account cleanup.
+// LaunchTemplateOwner is the launch-template owner's account scan, residue
+// cleanup and live-template delete; a run deletes live templates only when its
+// request asks and the consumer check completed.
 type LaunchTemplateOwner interface {
 	ScanAccount(ctx context.Context, accountID string) (*ec2launchtemplate.AccountScan, error)
 	CleanupAccountResidue(ctx context.Context, accountID string, settledBefore time.Time) (*ec2launchtemplate.AccountCleanup, error)
+	DeleteLiveTemplates(ctx context.Context, accountID string, referenced func(id, name string) string) (*ec2launchtemplate.AccountTemplateDeletes, error)
 }
 
 // TagIndexOwner lists and clears the account's tag-index entries.
@@ -69,6 +71,28 @@ var (
 	_ TagIndexOwner       = (*ec2tags.TagsServiceImpl)(nil)
 )
 
+// kindPrefixes maps each cleanable kind to its resource-ID prefix in the tag index.
+var kindPrefixes = map[string]string{
+	ec2v1.PlatformKindKeyPair:        keyPairIDPrefix,
+	ec2v1.PlatformKindPlacementGroup: placementGroupIDPrefix,
+	ec2v1.PlatformKindLaunchTemplate: launchTemplateIDPrefix,
+}
+
+// selectKinds returns the kinds a request cleans, in report order. No kinds
+// means all three; a kind that is not cleanable, such as the tag index, is refused.
+func selectKinds(requested []string) ([]string, error) {
+	all := []string{ec2v1.PlatformKindKeyPair, ec2v1.PlatformKindPlacementGroup, ec2v1.PlatformKindLaunchTemplate}
+	if len(requested) == 0 {
+		return all, nil
+	}
+	for _, kind := range requested {
+		if !slices.Contains(all, kind) {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The kind is not one of key-pair, placement-group or launch-template.")
+		}
+	}
+	return slices.DeleteFunc(all, func(kind string) bool { return !slices.Contains(requested, kind) }), nil
+}
+
 // DefaultSettleWindow is how old headerless residue must be before it is
 // removed; anything younger may be a create still in flight.
 const DefaultSettleWindow = 10 * time.Minute
@@ -102,6 +126,7 @@ const (
 // CleanupPlatform deletes the account's key pairs, empty placement groups and
 // launch templates, removes settled residue and orphaned tag-index entries,
 // then re-scans every owner store to build the result. It is safe to re-run.
+// Request.Kinds limits the run to those kinds and their tag-index entries.
 func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, error) {
 	if req == nil || req.AccountID == "" {
 		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "The request must specify an account ID.")
@@ -118,27 +143,59 @@ func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, e
 	acct := req.AccountID
 	res := &Result{AccountID: acct, CorrelationRef: req.CorrelationRef}
 
-	refs := c.readReferences(ctx, acct)
-	keyRun := c.cleanKeyPairs(ctx, acct, settledBefore, refs)
-	groupRun := c.cleanPlacementGroups(ctx, acct, refs)
-	templateRun := c.cleanLaunchTemplates(ctx, acct, settledBefore)
+	kinds, err := selectKinds(req.Kinds)
+	if err != nil {
+		return nil, err
+	}
+	var prefixes []string
+	for _, kind := range kinds {
+		prefixes = append(prefixes, kindPrefixes[kind])
+	}
+
+	var keyRun *keyPairRun
+	var groupRun *placementGroupRun
+	var templateRun *launchTemplateRun
+	// Live templates go first when asked: a version that names a key pair or
+	// group blocks it, so the reference read must see what remains.
+	if req.DeleteLiveTemplates && slices.Contains(kinds, ec2v1.PlatformKindLaunchTemplate) {
+		templateRun = c.cleanLaunchTemplates(ctx, acct, settledBefore, true)
+	}
+	if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) || slices.Contains(kinds, ec2v1.PlatformKindPlacementGroup) {
+		refs := c.readReferences(ctx, acct)
+		if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) {
+			keyRun = c.cleanKeyPairs(ctx, acct, settledBefore, refs)
+		}
+		if slices.Contains(kinds, ec2v1.PlatformKindPlacementGroup) {
+			groupRun = c.cleanPlacementGroups(ctx, acct, refs)
+		}
+	}
+	if templateRun == nil && slices.Contains(kinds, ec2v1.PlatformKindLaunchTemplate) {
+		templateRun = c.cleanLaunchTemplates(ctx, acct, settledBefore, false)
+	}
 
 	// Tags are listed before the owner re-scans, so a resource created in
 	// between is seen as owned and keeps its tags.
 	tagKind := c.startKind(ec2v1.PlatformKindTagIndex)
-	tagged, tagErr := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	tagged, tagErr := c.tags.ScanAccountResourceIDs(ctx, acct, prefixes...)
 	if tagErr != nil {
 		slog.WarnContext(ctx, "Platform cleanup: tag index listing failed", "err", tagErr)
 		tagKind.incomplete(ec2v1.PlatformReasonStoreListFailed)
 	}
 
-	owners := map[string]ownedIDs{
-		keyPairIDPrefix:        c.finishKeyPairs(ctx, acct, keyRun, res),
-		placementGroupIDPrefix: c.finishPlacementGroups(ctx, acct, groupRun, res),
-		launchTemplateIDPrefix: c.finishLaunchTemplates(ctx, acct, templateRun, res),
+	owners := map[string]ownedIDs{}
+	if keyRun != nil {
+		owners[keyPairIDPrefix] = c.finishKeyPairs(ctx, acct, keyRun, res)
+		res.Kinds = append(res.Kinds, keyRun.kind.report)
 	}
-	res.Kinds = []KindReport{keyRun.kind.report, groupRun.kind.report, templateRun.kind.report}
-	res.Kinds = append(res.Kinds, c.cleanTagIndex(ctx, acct, tagKind, tagged, tagErr == nil, owners, res))
+	if groupRun != nil {
+		owners[placementGroupIDPrefix] = c.finishPlacementGroups(ctx, acct, groupRun, res)
+		res.Kinds = append(res.Kinds, groupRun.kind.report)
+	}
+	if templateRun != nil {
+		owners[launchTemplateIDPrefix] = c.finishLaunchTemplates(ctx, acct, templateRun, res)
+		res.Kinds = append(res.Kinds, templateRun.kind.report)
+	}
+	res.Kinds = append(res.Kinds, c.cleanTagIndex(ctx, acct, tagKind, tagged, prefixes, tagErr == nil, owners, res))
 
 	slog.InfoContext(ctx, "Platform cleanup completed", "accountID", acct, "correlationRef", req.CorrelationRef,
 		"items", len(res.Items), "duration_ms", otelsetup.Millis(time.Since(start)))

spinifex/domains/ec2/accountcleanup/cleanup_live_templates_test.go

@@ -0,0 +1,159 @@
+package accountcleanup
+
+//test:in-package — shares the owner fixtures and the injected clock.
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeConsumers answers what nodegroups name; a non-nil err is returned with
+// whatever partial answer it also holds.
+type fakeConsumers struct {
+	ids, names []string
+	err        error
+}
+
+func (f *fakeConsumers) AccountTemplateConsumers(context.Context, string) (*TemplateConsumers, error) {
+	out := &TemplateConsumers{TemplateIDs: map[string]bool{}, TemplateNames: map[string]bool{}}
+	markAll(out.TemplateIDs, f.ids)
+	markAll(out.TemplateNames, f.names)
+	return out, f.err
+}
+
+var _ TemplateConsumerReader = (*fakeConsumers)(nil)
+
+// liveDelete runs a cleanup that deletes live templates, with consumers as the
+// nodegroup reader (nil means none supplied).
+func (o *owners) liveDelete(t *testing.T, consumers TemplateConsumerReader) *Result {
+	t.Helper()
+	readers := o.readers()
+	readers.Consumers = consumers
+	c := New(o.keys, o.groups, o.templates, o.tags, readers)
+	c.now = func() time.Time { return time.Now().Add(time.Hour) }
+	res, err := c.CleanupPlatform(t.Context(), &Request{AccountID: testAccount, CorrelationRef: "ref-1", DeleteLiveTemplates: true})
+	require.NoError(t, err)
+	return res
+}
+
+func (o *owners) templateNamingKeyAndGroup(t *testing.T, name, key, group string) string {
+	t.Helper()
+	ltID := o.createTemplate(t, testAccount, name)
+	_, err := o.templates.CreateLaunchTemplateVersion(t.Context(), &ec2.CreateLaunchTemplateVersionInput{
+		LaunchTemplateId: aws.String(ltID),
+		LaunchTemplateData: &ec2.RequestLaunchTemplateData{
+			KeyName:   aws.String(key),
+			Placement: &ec2.LaunchTemplatePlacementRequest{GroupName: aws.String(group)},
+		},
+	}, testAccount)
+	require.NoError(t, err)
+	return ltID
+}
+
+func (o *owners) liveTemplateIDs(t *testing.T) []string {
+	t.Helper()
+	scan, err := o.templates.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	return scan.LiveTemplateIDs
+}
+
+// A template no nodegroup names is deleted in the same run as the key pair and
+// group its version names, which the reference check no longer sees.
+func TestCleanupPlatformDeletesAnUnnamedTemplateAndWhatItNames(t *testing.T) {
+	o := newOwners(t)
+	key := o.importKey(t, testAccount, "tkey")
+	group := o.createGroup(t, testAccount, "tgroup")
+	lt := o.templateNamingKeyAndGroup(t, "web", "tkey", "tgroup")
+
+	res := o.liveDelete(t, &fakeConsumers{})
+	assert.ElementsMatch(t, []Item{
+		item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionDeleted, ""),
+		item(key, ec2v1.PlatformKindKeyPair, DispositionDeleted, ""),
+		item(group, ec2v1.PlatformKindPlacementGroup, DispositionDeleted, ""),
+	}, res.Items)
+	assert.Empty(t, o.liveTemplateIDs(t))
+}
+
+// A template a nodegroup names, by ID or by name, is kept and blocked, and so
+// are the key pair and group its versions name.
+func TestCleanupPlatformKeepsATemplateANodegroupNames(t *testing.T) {
+	for name, consumers := range map[string]*fakeConsumers{
+		"by id":   {},
+		"by name": {names: []string{"web"}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			o := newOwners(t)
+			key := o.importKey(t, testAccount, "tkey")
+			group := o.createGroup(t, testAccount, "tgroup")
+			lt := o.templateNamingKeyAndGroup(t, "web", "tkey", "tgroup")
+			other := o.createTemplate(t, testAccount, "free")
+			if name == "by id" {
+				consumers.ids = []string{lt}
+			}
+
+			res := o.liveDelete(t, consumers)
+			assert.ElementsMatch(t, []Item{
+				item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonLaunchTemplateReference),
+				item(other, ec2v1.PlatformKindLaunchTemplate, DispositionDeleted, ""),
+				item(key, ec2v1.PlatformKindKeyPair, DispositionBlocked, ReasonLaunchTemplateReference),
+				item(group, ec2v1.PlatformKindPlacementGroup, DispositionBlocked, ReasonLaunchTemplateReference),
+			}, res.Items)
+			assert.Equal(t, []string{lt}, o.liveTemplateIDs(t))
+		})
+	}
+}
+
+// A consumer read that fails, returns a partial answer with an error, or is not
+// supplied deletes no template, key pair or group, and the kind is incomplete.
+func TestCleanupPlatformUnreadableConsumersDeleteNoTemplate(t *testing.T) {
+	cases := map[string]TemplateConsumerReader{
+		"error":        &fakeConsumers{err: errInjected},
+		"partial":      &fakeConsumers{ids: []string{"lt-0other"}, err: errInjected},
+		"not supplied": nil,
+	}
+	for name, consumers := range cases {
+		t.Run(name, func(t *testing.T) {
+			o := newOwners(t)
+			key := o.importKey(t, testAccount, "tkey")
+			lt := o.templateNamingKeyAndGroup(t, "web", "tkey", "none")
+
+			res := o.liveDelete(t, consumers)
+			assert.Contains(t, res.Items, item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonReferenceCheckFailed))
+			assert.Contains(t, res.Items, item(key, ec2v1.PlatformKindKeyPair, DispositionBlocked, ReasonLaunchTemplateReference))
+			assert.Equal(t, ec2v1.PlatformStatusScanIncomplete, statuses(res)[ec2v1.PlatformKindLaunchTemplate])
+			assert.Equal(t, []string{lt}, o.liveTemplateIDs(t))
+		})
+	}
+}
+
+// A template only a DELETING nodegroup still names is as blocked as any other:
+// the reader reports every status, so the cleanup never sees a status to trust.
+func TestCleanupPlatformKeepsATemplateADeletingNodegroupStillNames(t *testing.T) {
+	o := newOwners(t)
+	lt := o.createTemplate(t, testAccount, "draining")
+
+	res := o.liveDelete(t, &fakeConsumers{ids: []string{lt}})
+	assert.Equal(t, []Item{item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonLaunchTemplateReference)}, res.Items)
+	assert.Equal(t, []string{lt}, o.liveTemplateIDs(t))
+}
+
+// Without the request flag a supplied consumer reader changes nothing: live
+// templates are never deleted, as the exercise relies on.
+func TestCleanupPlatformLiveTemplatesSurviveWithoutTheRequestFlag(t *testing.T) {
+	o := newOwners(t)
+	lt := o.createTemplate(t, testAccount, "web")
+	readers := o.readers()
+	readers.Consumers = &fakeConsumers{}
+	c := New(o.keys, o.groups, o.templates, o.tags, readers)
+
+	res := run(t, c)
+	assert.Equal(t, []Item{item(lt, ec2v1.PlatformKindLaunchTemplate, DispositionBlocked, ReasonReferenceUnknown)}, res.Items)
+	assert.Equal(t, []string{lt}, o.liveTemplateIDs(t))
+}

spinifex/domains/ec2/accountcleanup/cleanup_test.go

@@ -775,3 +775,38 @@ func (f failingResidue) CleanupAccountResidue(ctx context.Context, acct string,
 	}
 	return &ec2launchtemplate.AccountCleanup{KeptTemplateIDs: scan.LiveTemplateIDs, OrphanBodiesFailed: scan.OrphanVersionBodies}, nil
 }
+
+// A run limited to one kind cleans that kind and its tag-index entries only:
+// the other kinds keep their resources and their tags, and are not reported.
+func TestCleanupPlatformLimitedToKindsLeavesTheRestAlone(t *testing.T) {
+	o := newOwners(t)
+	keyID := o.importKey(t, testAccount, "k1")
+	groupID := o.createGroup(t, testAccount, "g1")
+	o.tag(t, testAccount, keyID)
+
+	c := o.newCleanup()
+	c.now = func() time.Time { return time.Now().Add(time.Hour) }
+	res, err := c.CleanupPlatform(t.Context(), &Request{AccountID: testAccount, Kinds: []string{ec2v1.PlatformKindPlacementGroup}})
+	require.NoError(t, err)
+
+	assert.Equal(t, []Item{item(groupID, ec2v1.PlatformKindPlacementGroup, DispositionDeleted, "")}, res.Items)
+	assert.Equal(t, map[string]string{
+		ec2v1.PlatformKindPlacementGroup: ec2v1.PlatformStatusObservedClean,
+		ec2v1.PlatformKindTagIndex:       ec2v1.PlatformStatusObservedClean,
+	}, statuses(res))
+
+	keys, err := o.keys.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, keys.LiveKeyPairIDs, "an unselected kind is not touched")
+	tagged, err := o.tags.ScanAccountResourceIDs(t.Context(), testAccount, "key-")
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, tagged, "an unselected kind keeps its tags")
+}
+
+func TestCleanupPlatformRefusesAKindItCannotClean(t *testing.T) {
+	o := newOwners(t)
+	for _, kinds := range [][]string{{ec2v1.PlatformKindTagIndex}, {"volume"}} {
+		_, err := o.newCleanup().CleanupPlatform(t.Context(), &Request{AccountID: testAccount, Kinds: kinds})
+		require.Error(t, err, "%v", kinds)
+	}
+}

spinifex/domains/ec2/accountcleanup/kinds.go

@@ -204,13 +204,18 @@ type launchTemplateRun struct {
 	before  *ec2launchtemplate.AccountScan
 	cleaned *ec2launchtemplate.AccountCleanup
 	did     attempts
+	// deleteLive is true when this run tried to delete live templates.
+	deleteLive bool
 }
 
-func (c *Cleanup) cleanLaunchTemplates(ctx context.Context, acct string, settledBefore time.Time) *launchTemplateRun {
-	run := &launchTemplateRun{kind: c.startKind(ec2v1.PlatformKindLaunchTemplate), did: newAttempts()}
+func (c *Cleanup) cleanLaunchTemplates(ctx context.Context, acct string, settledBefore time.Time, deleteLive bool) *launchTemplateRun {
+	run := &launchTemplateRun{kind: c.startKind(ec2v1.PlatformKindLaunchTemplate), did: newAttempts(), deleteLive: deleteLive}
 	if before, err := c.templates.ScanAccount(ctx, acct); err == nil {
 		run.before = before
 	}
+	if deleteLive {
+		c.deleteLiveTemplates(ctx, acct, run)
+	}
 	cleaned, err := c.templates.CleanupAccountResidue(ctx, acct, settledBefore)
 	if err != nil {
 		slog.WarnContext(ctx, "Platform cleanup: launch-template residue cleanup failed", "err", err)
@@ -218,10 +223,50 @@ func (c *Cleanup) cleanLaunchTemplates(ctx context.Context, acct string, settled
 		return run
 	}
 	run.cleaned = cleaned
-	keepAll(run.did.kept, cleaned.KeptTemplateIDs, ReasonReferenceUnknown)
+	for _, id := range cleaned.KeptTemplateIDs {
+		if run.did.kept[id] == "" {
+			run.did.kept[id] = ReasonReferenceUnknown
+		}
+	}
 	return run
 }
 
+// deleteLiveTemplates deletes the live templates no consumer names. Without a
+// complete consumer answer nothing is deleted and every live template is kept
+// as unchecked, so an unreadable consumer never reads as "no references".
+func (c *Cleanup) deleteLiveTemplates(ctx context.Context, acct string, run *launchTemplateRun) {
+	var consumers *TemplateConsumers
+	if c.refs.Consumers != nil {
+		var err error
+		consumers, err = c.refs.Consumers.AccountTemplateConsumers(ctx, acct)
+		if err != nil {
+			slog.WarnContext(ctx, "Platform cleanup: template consumer check failed", "err", err)
+			consumers = nil
+		}
+	}
+	if consumers == nil {
+		if run.before != nil {
+			keepAll(run.did.kept, run.before.LiveTemplateIDs, ReasonReferenceCheckFailed)
+		}
+		run.kind.incomplete(ReasonReferenceCheckFailed)
+		return
+	}
+	deletes, err := c.templates.DeleteLiveTemplates(ctx, acct, func(id, name string) string {
+		if consumers.names(id, name) {
+			return ReasonLaunchTemplateReference
+		}
+		return ""
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "Platform cleanup: live template delete failed", "err", err)
+		run.kind.incomplete(ec2v1.PlatformReasonStoreListFailed)
+		return
+	}
+	markAll(run.did.deleted, deletes.DeletedTemplateIDs)
+	markAll(run.did.failed, deletes.FailedTemplateIDs)
+	maps.Copy(run.did.kept, deletes.Referenced)
+}
+
 func (c *Cleanup) finishLaunchTemplates(ctx context.Context, acct string, run *launchTemplateRun, res *Result) ownedIDs {
 	defer c.finishKind(run.kind)
 	after, err := c.templates.ScanAccount(ctx, acct)
@@ -250,8 +295,10 @@ func (c *Cleanup) finishLaunchTemplates(ctx context.Context, acct string, run *l
 	}
 	// Live templates are never deleted here, so every one still present is
 	// blocked on the unknown EKS nodegroup reference, whenever it appeared.
-	for _, id := range afterLive {
-		run.did.kept[id] = ReasonReferenceUnknown
+	if !run.deleteLive {
+		for _, id := range afterLive {
+			run.did.kept[id] = ReasonReferenceUnknown
+		}
 	}
 	for _, id := range sortedUnion(beforeLive, kept, afterLive) {
 		disposition, reason := itemDisposition(id, run.did, present[id], after != nil)
@@ -282,15 +329,15 @@ func (c *Cleanup) finishLaunchTemplates(ctx context.Context, acct string, run *l
 // cleanTagIndex clears entries whose owner re-scan resolved and holds no
 // record for the ID, then re-lists the index to prove each clear. Entries of
 // resources that still exist are theirs, not residue, and are left alone.
-func (c *Cleanup) cleanTagIndex(ctx context.Context, acct string, kind *kindRun, tagged []string, listed bool, owners map[string]ownedIDs, res *Result) KindReport {
+func (c *Cleanup) cleanTagIndex(ctx context.Context, acct string, kind *kindRun, tagged, prefixes []string, listed bool, owners map[string]ownedIDs, res *Result) KindReport {
 	if listed {
-		c.clearOrphanTags(ctx, acct, kind, tagged, owners, res)
+		c.clearOrphanTags(ctx, acct, kind, tagged, prefixes, owners, res)
 	}
 	c.finishKind(kind)
 	return kind.report
 }
 
-func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRun, tagged []string, owners map[string]ownedIDs, res *Result) {
+func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRun, tagged, prefixes []string, owners map[string]ownedIDs, res *Result) {
 	// A resource this run reports as not deleted owns its tags even when its
 	// kind's re-scan failed.
 	reported := map[string]bool{}
@@ -317,7 +364,7 @@ func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRu
 			cleared = append(cleared, id)
 		}
 	}
-	after, err := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	after, err := c.tags.ScanAccountResourceIDs(ctx, acct, prefixes...)
 	if err != nil {
 		slog.WarnContext(ctx, "Platform cleanup: tag index re-scan failed", "err", err)
 		kind.incomplete(ec2v1.PlatformReasonStoreListFailed)

spinifex/domains/ec2/accountcleanup/references.go

@@ -35,17 +35,38 @@ type TemplateReferenceReader interface {
 	AccountTemplateReferences(ctx context.Context, accountID string) (*ec2launchtemplate.TemplateReferences, error)
 }
 
+// TemplateConsumers are the IDs and names of launch templates that records
+// outside EC2, such as EKS nodegroups, still name. They select what to keep and
+// are never reported.
+type TemplateConsumers struct {
+	TemplateIDs   map[string]bool
+	TemplateNames map[string]bool
+}
+
+// TemplateConsumerReader reads which launch templates records outside EC2 name,
+// every status included, complete or an error. The consumer's owner answers it;
+// EC2 never reads those records.
+type TemplateConsumerReader interface {
+	AccountTemplateConsumers(ctx context.Context, accountID string) (*TemplateConsumers, error)
+}
+
+func (c *TemplateConsumers) names(id, name string) bool {
+	return c.TemplateIDs[id] || c.TemplateNames[name]
+}
+
 var (
 	_ SpotReferenceReader     = (*ec2spotinstance.SpotInstanceServiceImpl)(nil)
 	_ TemplateReferenceReader = (*ec2launchtemplate.LaunchTemplateServiceImpl)(nil)
 )
 
 // ReferenceReaders are the checks run before key pairs and placement groups
-// are deleted. A nil reader counts as a failed check.
+// are deleted. A nil reader counts as a failed check. Consumers is read only
+// by a request that deletes live templates, where nil is also a failed check.
 type ReferenceReaders struct {
 	Instances InstanceReferenceReader
 	Spot      SpotReferenceReader
 	Templates TemplateReferenceReader
+	Consumers TemplateConsumerReader
 }
 
 func newReferences() *References {

spinifex/domains/ec2/accountcleanup/types.go

@@ -7,6 +7,14 @@ import "time"
 type Request struct {
 	AccountID      string `json:"account_id"`
 	CorrelationRef string `json:"correlation_ref,omitempty"`
+
+	// Kinds limits the run to these platform kinds. Empty means all three.
+	Kinds []string `json:"kinds,omitempty"`
+
+	// DeleteLiveTemplates lets the run delete live launch templates that no
+	// nodegroup names, using the TemplateConsumerReader. Without it live
+	// templates are never deleted.
+	DeleteLiveTemplates bool `json:"delete_live_templates,omitempty"`
 }
 
 // Result is what the owner stores hold after one run. Kinds reuse the ec2.v1
@@ -85,7 +93,8 @@ const (
 	ReasonInstanceReference = "instance-reference"
 	// ReasonSpotRequestReference: an open or active spot request names it.
 	ReasonSpotRequestReference = "spot-request-reference"
-	// ReasonLaunchTemplateReference: a version of a live launch template names it.
+	// ReasonLaunchTemplateReference: a version of a live launch template names a
+	// key pair or group, or a nodegroup names a template.
 	ReasonLaunchTemplateReference = "launch-template-reference"
 	// ReasonReferenceUnknown: a record EC2 cannot read may name it, so it is
 	// never deleted by account cleanup.

spinifex/domains/ec2/awsapi/registration.go

@@ -1,8 +1,6 @@
 package awsapi
 
 import (
-	"net/http"
-
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 )
 
@@ -11,7 +9,7 @@ import (
 func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
-		ResolveAction: resolveAction,
+		ResolveAction: dispatch.ResolveQueryAction,
 		Dispatch:      NewHandler(deps).Dispatch,
 		Errors:        dispatch.ErrorEnvelopeXML,
 		Traits:        dispatch.ServiceTraits{XMLErrorShape: dispatch.XMLErrorShapeEC2},
@@ -24,13 +22,3 @@ func NewRegistration(deps Deps) dispatch.Registration {
 func OperationInventory() dispatch.Inventory {
 	return dispatch.Inventory{Registered: ActionNames()}
 }
-
-// resolveAction is the one Query Action parser telemetry and dispatch share.
-// It leaves the body readable for the dispatcher and answers "" on any failure.
-func resolveAction(r *http.Request) string {
-	args, err := readQueryArgs(r)
-	if err != nil {
-		return ""
-	}
-	return args["Action"]
-}

spinifex/domains/ec2/awsapi/registration_test.go

@@ -1,11 +1,6 @@
-//test:in-package — drives the unexported resolveAction with the registration.
-
 package awsapi
 
 import (
-	"net/http"
-	"net/http/httptest"
-	"strings"
 	"testing"
 
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -27,29 +22,3 @@ func TestNewRegistration_DeclaresTheEC2Service(t *testing.T) {
 	b := dispatch.NewBuilder()
 	require.NoError(t, b.Register(reg))
 }
-
-func TestResolveAction(t *testing.T) {
-	post := func(body string) *http.Request {
-		r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
-		r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-		return r
-	}
-
-	assert.Equal(t, "DescribeVpcs", resolveAction(post("Action=DescribeVpcs&Version=2016-11-15")))
-	assert.Empty(t, resolveAction(post("Version=2016-11-15")))
-
-	// The body stays readable for the dispatcher after the action is resolved.
-	r := post("Action=DescribeVpcs")
-	require.Equal(t, "DescribeVpcs", resolveAction(r))
-	assert.Equal(t, "DescribeVpcs", resolveAction(r))
-}
-
-func TestResolveAction_UnreadableBody(t *testing.T) {
-	r := httptest.NewRequest(http.MethodPost, "/", errReader{})
-	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	assert.Empty(t, resolveAction(r))
-}
-
-type errReader struct{}
-
-func (errReader) Read([]byte) (int, error) { return 0, http.ErrBodyReadAfterClose }

spinifex/domains/ec2/awsapi/request.go

@@ -1,9 +1,7 @@
 package awsapi
 
 import (
-	"bytes"
 	"errors"
-	"io"
 	"log/slog"
 	"maps"
 	"net/http"
@@ -12,7 +10,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/bluebottle/pkg/iampolicy"
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -26,25 +23,11 @@ const (
 	ec2ServicePrincipal = "ec2.amazonaws.com"
 )
 
-// readQueryArgs parses the Query body and rewinds it, so the same bytes can be
-// read again. Any read or parse failure is a malformed query string.
-func readQueryArgs(r *http.Request) (map[string]string, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, err
-	}
-	r.Body = io.NopCloser(bytes.NewReader(body))
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return query.ParseArgs(string(body))
-}
-
 // Dispatch parses, authorizes and runs one EC2 request, then writes the XML
 // response. A returned error is an awserrors code the gateway renders.
 func (h *Handler) Dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
-	queryArgs, err := readQueryArgs(r)
+	queryArgs, err := dispatch.ReadArgs(r)
 	if err != nil {
 		slog.Debug("EC2: malformed query string", "err", err)
 		return errors.New(awserrors.ErrorMalformedQueryString)

spinifex/domains/ec2/awsapi/volume/AttachVolume.go

@@ -49,6 +49,12 @@ func AttachVolume(ctx context.Context, input *ec2.AttachVolumeInput, natsConn *n
 		device = *input.Device
 	}
 
+	// The shared record decides: a stopped instance has no QMP to plug into, so
+	// its record is edited and the volume claimed there instead.
+	if isStoppedInstance(ctx, instanceID, natsConn, accountID) {
+		return attachToStopped(ctx, natsConn, accountID, input)
+	}
+
 	command := ec2v1.EC2InstanceCommand{
 		ID: instanceID,
 		Attributes: ec2v1.EC2CommandAttributes{
@@ -97,6 +103,24 @@ func AttachVolume(ctx context.Context, input *ec2.AttachVolumeInput, natsConn *n
 	return output, nil
 }
 
+// attachToStoppedSubject is answered by any node and edits the stopped
+// instance record in the shared bucket.
+const attachToStoppedSubject = "ec2.AttachVolumeToStopped"
+
+// attachToStopped asks the instance service to add the volume to the stopped
+// instance's record; the volume is claimed in-use by the same call.
+func attachToStopped(ctx context.Context, natsConn *nats.Conn, accountID string, input *ec2.AttachVolumeInput) (ec2.VolumeAttachment, error) {
+	attachment, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, natsConn, attachToStoppedSubject, input, 30*time.Second, accountID)
+	if err != nil {
+		slog.ErrorContext(ctx, "AttachVolume: stopped-instance attach failed", "volumeId", aws.StringValue(input.VolumeId), "instanceId", aws.StringValue(input.InstanceId), "err", err)
+		return ec2.VolumeAttachment{}, err
+	}
+	if attachment == nil {
+		return ec2.VolumeAttachment{}, nil
+	}
+	return *attachment, nil
+}
+
 // isStoppedInstance checks the shared KV (via ec2.DescribeStoppedInstances) to
 // determine whether instanceID exists as a stopped instance.
 func isStoppedInstance(ctx context.Context, instanceID string, natsConn *nats.Conn, accountID string) bool {

spinifex/domains/ec2/awsapi/volume/AttachVolume_stopped_test.go

@@ -0,0 +1,55 @@
+package volume
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The record decides the path: a stopped instance has no QMP to plug into, so
+// the attach goes to the instance service and ec2.cmd.<id> is never used.
+func TestAttachVolume_StoppedInstanceUsesTheRecordPath(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	log := &subjectLog{}
+
+	stoppedRecord := &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{
+		Instances: []*ec2.Instance{{InstanceId: aws.String("i-stopped")}},
+	}}}
+	handlers := map[string]func(*nats.Msg){
+		"ec2.DescribeStoppedInstances": func(m *nats.Msg) { respondJSON(t, m, stoppedRecord) },
+		attachToStoppedSubject: func(m *nats.Msg) {
+			log.add(attachToStoppedSubject)
+			respondJSON(t, m, &ec2.VolumeAttachment{
+				VolumeId: aws.String("vol-data"), InstanceId: aws.String("i-stopped"),
+				Device: aws.String("/dev/sdf"), State: aws.String("attached"),
+			})
+		},
+		"ec2.cmd.i-stopped": func(m *nats.Msg) {
+			log.add("ec2.cmd.i-stopped")
+			_ = m.Respond(nil)
+		},
+	}
+	for subject, h := range handlers {
+		sub, err := nc.Subscribe(subject, h)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	require.NoError(t, nc.Flush())
+
+	out, err := AttachVolume(context.Background(), &ec2.AttachVolumeInput{
+		VolumeId:   aws.String("vol-data"),
+		InstanceId: aws.String("i-stopped"),
+		Device:     aws.String("/dev/sdf"),
+	}, nc, "000000000001")
+	require.NoError(t, err)
+
+	assert.Equal(t, "attached", aws.StringValue(out.State))
+	assert.Equal(t, []string{attachToStoppedSubject}, log.get(),
+		"only the record path runs for a stopped instance")
+}

spinifex/domains/ec2/awsapi/volume/DetachVolume.go

@@ -74,6 +74,12 @@ func DetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, natsConn *n
 
 	force := input.Force != nil && *input.Force
 
+	// The shared record decides: a stopped instance has no QMP to unplug from,
+	// so its record is edited first and the volume released after.
+	if isStoppedInstance(ctx, instanceID, natsConn, accountID) {
+		return detachFromStopped(ctx, natsConn, accountID, volumeID, instanceID, device)
+	}
+
 	command := ec2v1.EC2InstanceCommand{
 		ID: instanceID,
 		Attributes: ec2v1.EC2CommandAttributes{
@@ -131,6 +137,10 @@ func DetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, natsConn *n
 // running the instance and so cannot help when that host is the problem.
 const forceDetachSubject = "ec2.ForceDetachVolume"
 
+// detachFromStoppedSubject is answered by any node and edits the stopped
+// instance record in the shared bucket.
+const detachFromStoppedSubject = "ec2.DetachVolumeFromStopped"
+
 // forceDetach releases a volume whose instance has no responder.
 //
 // A volume outlives the instance it was attached to whenever teardown failed
@@ -153,3 +163,27 @@ func forceDetach(ctx context.Context, natsConn *nats.Conn, accountID, volumeID,
 	}
 	return *attachment, nil
 }
+
+// detachFromStopped removes the volume from the stopped instance's record and
+// then releases the volume, in that order, so a failed release leaves the
+// record clean and the volume still held rather than the reverse.
+func detachFromStopped(ctx context.Context, natsConn *nats.Conn, accountID, volumeID, instanceID, device string) (ec2.VolumeAttachment, error) {
+	in := &ec2.DetachVolumeInput{VolumeId: &volumeID, InstanceId: &instanceID}
+	if device != "" {
+		in.Device = &device
+	}
+	if _, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, natsConn, detachFromStoppedSubject, in, 30*time.Second, accountID); err != nil {
+		slog.ErrorContext(ctx, "DetachVolume: stopped-instance record edit failed", "volumeId", volumeID, "instanceId", instanceID, "err", err)
+		return ec2.VolumeAttachment{}, err
+	}
+	attachment, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, natsConn, forceDetachSubject,
+		&ec2.DetachVolumeInput{VolumeId: &volumeID, Force: aws.Bool(true)}, 30*time.Second, accountID)
+	if err != nil {
+		slog.ErrorContext(ctx, "DetachVolume: release after record edit failed", "volumeId", volumeID, "instanceId", instanceID, "err", err)
+		return ec2.VolumeAttachment{}, err
+	}
+	if attachment == nil {
+		return ec2.VolumeAttachment{}, nil
+	}
+	return *attachment, nil
+}

spinifex/domains/ec2/awsapi/volume/DetachVolume_stopped_test.go

@@ -0,0 +1,115 @@
+package volume
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type subjectLog struct {
+	mu   sync.Mutex
+	seen []string
+}
+
+func (l *subjectLog) add(s string) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.seen = append(l.seen, s)
+}
+
+func (l *subjectLog) get() []string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return append([]string(nil), l.seen...)
+}
+
+func respondJSON(t *testing.T, msg *nats.Msg, v any) {
+	t.Helper()
+	data, err := json.Marshal(v)
+	require.NoError(t, err)
+	require.NoError(t, msg.Respond(data))
+}
+
+// The record decides the path. A stopped instance has no QMP to unplug from, so
+// its record is edited and the volume released, and ec2.cmd.<id> is never used.
+func TestDetachVolume_StoppedInstanceEditsTheRecordThenReleasesTheVolume(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	log := &subjectLog{}
+
+	stoppedRecord := &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{
+		Instances: []*ec2.Instance{{InstanceId: aws.String("i-stopped")}},
+	}}}
+	handlers := map[string]func(*nats.Msg){
+		"ec2.DescribeStoppedInstances": func(m *nats.Msg) { respondJSON(t, m, stoppedRecord) },
+		detachFromStoppedSubject: func(m *nats.Msg) {
+			log.add(detachFromStoppedSubject)
+			respondJSON(t, m, &ec2.VolumeAttachment{
+				VolumeId: aws.String("vol-data"), InstanceId: aws.String("i-stopped"), Device: aws.String("/dev/sdf"),
+			})
+		},
+		forceDetachSubject: func(m *nats.Msg) {
+			log.add(forceDetachSubject)
+			respondJSON(t, m, &ec2.VolumeAttachment{
+				VolumeId: aws.String("vol-data"), InstanceId: aws.String("i-stopped"), State: aws.String("detached"),
+			})
+		},
+		"ec2.cmd.i-stopped": func(m *nats.Msg) {
+			log.add("ec2.cmd.i-stopped")
+			_ = m.Respond(nil)
+		},
+	}
+	for subject, h := range handlers {
+		sub, err := nc.Subscribe(subject, h)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	require.NoError(t, nc.Flush())
+
+	out, err := DetachVolume(context.Background(), &ec2.DetachVolumeInput{
+		VolumeId:   aws.String("vol-data"),
+		InstanceId: aws.String("i-stopped"),
+	}, nc, "000000000001")
+	require.NoError(t, err)
+
+	assert.Equal(t, "detached", aws.StringValue(out.State))
+	assert.Equal(t, []string{detachFromStoppedSubject, forceDetachSubject}, log.get(),
+		"record first, then the volume; the host path is never used for a stopped instance")
+}
+
+// NoResponders alone must not be read as "stopped": a dead or partitioned node
+// gives the same answer, and editing the record then would race a live QEMU.
+func TestDetachVolume_NoRespondersAloneDoesNotMeanStopped(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	log := &subjectLog{}
+
+	for _, subject := range []string{detachFromStoppedSubject, forceDetachSubject} {
+		sub, err := nc.Subscribe(subject, func(m *nats.Msg) {
+			log.add(subject)
+			_ = m.Respond(nil)
+		})
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	sub, err := nc.Subscribe("ec2.DescribeStoppedInstances", func(m *nats.Msg) {
+		respondJSON(t, m, &ec2.DescribeInstancesOutput{})
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, nc.Flush())
+
+	_, err = DetachVolume(context.Background(), &ec2.DetachVolumeInput{
+		VolumeId:   aws.String("vol-data"),
+		InstanceId: aws.String("i-silent-node"),
+	}, nc, "000000000001")
+
+	require.Error(t, err)
+	assert.Empty(t, log.get(), "no record edit and no control-plane release without a stopped record")
+}

spinifex/domains/ec2/image/drain_test.go

@@ -1,140 +0,0 @@
-package image
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const drainImageInstanceID = "i-image-drain-host"
-
-// setupDrainImageService mirrors domains/ec2/snapshot's setupDrainService: a
-// reachable (but rejecting) Predastore host so a drained volume falls through
-// into the real snapshot path and fails fast, a data dir with no local drain
-// socket, and a NATS connection to route the drain to the hosting node.
-func setupDrainImageService(t *testing.T) (*ImageServiceImpl, *objectstore.MemoryObjectStore, *nats.Conn) {
-	t.Helper()
-
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	t.Cleanup(srv.Close)
-
-	_, nc := testutil.StartTestNATS(t)
-	store := objectstore.NewMemoryObjectStore()
-	cfg := &config.Config{
-		DataDir: testutil.SocketTempDir(t),
-		Predastore: config.PredastoreConfig{
-			Host:   srv.URL,
-			Bucket: testBucket,
-		},
-	}
-	svc := NewImageServiceImplWithConfig(cfg, store, nc)
-	return svc, store, nc
-}
-
-func seedImageVolumeAttachment(t *testing.T, store *objectstore.MemoryObjectStore, volumeID, state, instanceID string) {
-	t.Helper()
-	require.NoError(t, volumestate.Write(context.Background(), store, testBucket, volumeID, volumestate.Record{
-		State:            state,
-		AttachedInstance: instanceID,
-	}))
-}
-
-func drainImageResponder(t *testing.T, nc *nats.Conn, instanceID string, reply []byte) chan *nats.Msg {
-	t.Helper()
-
-	got := make(chan *nats.Msg, 4)
-	sub, err := nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
-		got <- msg
-		_ = msg.Respond(reply)
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-
-	return got
-}
-
-func drainedImageAck(t *testing.T, volumeID string) []byte {
-	t.Helper()
-	data, err := json.Marshal(ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusDrained})
-	require.NoError(t, err)
-	return data
-}
-
-func awaitImageDrainCommand(t *testing.T, got chan *nats.Msg) ec2v1.EC2InstanceCommand {
-	t.Helper()
-	select {
-	case msg := <-got:
-		var command ec2v1.EC2InstanceCommand
-		require.NoError(t, json.Unmarshal(msg.Data, &command))
-		return command
-	case <-time.After(2 * time.Second):
-		t.Fatal("the node hosting the volume never received a drain command")
-		return ec2v1.EC2InstanceCommand{}
-	}
-}
-
-// CreateImageFromInstance's IsRunning path must drain the volume's hosting node
-// before it reads the live checkpoint — the same guarantee ec2.CreateSnapshot
-// has. Without this, a checkpoint predating a guest-triggered GPT rewrite
-// (e.g. growpart on first boot) can be captured silently.
-func TestSnapshotRunningVolume_DrainsAttachedVolume(t *testing.T) {
-	svc, store, nc := setupDrainImageService(t)
-	createTestVolumeConfig(t, store, "vol-image-drain", 10)
-	seedImageVolumeAttachment(t, store, "vol-image-drain", "in-use", drainImageInstanceID)
-	got := drainImageResponder(t, nc, drainImageInstanceID, drainedImageAck(t, "vol-image-drain"))
-
-	err := svc.snapshotRunningVolume("vol-image-drain", "snap-image-drain", testAccountID)
-	// The rejecting Predastore stub fails the snapshot itself; what matters
-	// here is that the drain reached the hosting node first.
-	require.Error(t, err)
-
-	command := awaitImageDrainCommand(t, got)
-	assert.True(t, command.Attributes.DrainVolume)
-	assert.Equal(t, drainImageInstanceID, command.ID)
-	require.NotNil(t, command.DrainVolumeData)
-	assert.Equal(t, "vol-image-drain", command.DrainVolumeData.VolumeID)
-}
-
-// An attached volume whose host reports the drain failed must fail
-// CreateImage rather than register an AMI built from a stale checkpoint.
-func TestSnapshotRunningVolume_UndrainableAttachedVolumeFails(t *testing.T) {
-	svc, store, nc := setupDrainImageService(t)
-	createTestVolumeConfig(t, store, "vol-image-nodrain", 10)
-	seedImageVolumeAttachment(t, store, "vol-image-nodrain", "in-use", drainImageInstanceID)
-	drainImageResponder(t, nc, drainImageInstanceID,
-		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
-
-	err := svc.snapshotRunningVolume("vol-image-nodrain", "snap-image-nodrain", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// An unattached (stopped-instance) volume takes the no-drain path: the
-// checkpoint Close() left behind is already current.
-func TestSnapshotRunningVolume_AvailableSkipsDrain(t *testing.T) {
-	svc, store, _ := setupDrainImageService(t)
-	createTestVolumeConfig(t, store, "vol-image-available", 10)
-	seedImageVolumeAttachment(t, store, "vol-image-available", "available", "")
-	svc.natsConn = nil
-
-	// No responder exists and the socket is never dialled: proceeding straight
-	// to the (rejecting) snapshot path proves the drain was skipped, not stuck.
-	err := svc.snapshotRunningVolume("vol-image-available", "snap-image-available", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}

spinifex/domains/ec2/image/provider_snapshot_test.go

@@ -2,16 +2,20 @@ package image
 
 import (
 	"context"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -96,15 +100,15 @@ func TestGetVolumeMetadata_Provider_UnknownVolumeIsNotFound(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error())
 }
 
-// TestSnapshotStoppedVolume_Provider_CreatesThroughProvider locks that a
-// stopped-volume snapshot under a provider is created via provider.CreateSnapshot
+// TestSnapshotVolume_Provider_CreatesThroughProvider locks that a
+// root-volume snapshot under a provider is created via provider.CreateSnapshot
 // and never falls through to building an embedded engine — the Predastore host
 // is a closed port, so any fallback would fail with connection refused.
-func TestSnapshotStoppedVolume_Provider_CreatesThroughProvider(t *testing.T) {
+func TestSnapshotVolume_Provider_CreatesThroughProvider(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-stopped001", 8)
 
-	err := svc.snapshotStoppedVolume(doc.VolumeID, "snap-stopped001", testAccountID)
+	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped001", testAccountID)
 	require.NoError(t, err, "the provider path must not fall through to the embedded engine against the closed Predastore port")
 
 	// CreateSnapshot is idempotent on (SnapshotID, VolumeID): re-issuing it
@@ -118,22 +122,22 @@ func TestSnapshotStoppedVolume_Provider_CreatesThroughProvider(t *testing.T) {
 	assert.Equal(t, doc.VolumeID, snap.SourceVolumeID)
 }
 
-// TestSnapshotRunningVolume_Provider_PassesProviderHandleAsVolumeHandle locks
+// TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle locks
 // that the ebsmetadata document's ProviderHandle is threaded through as
 // CreateSnapshotRequest.VolumeHandle. A wrong handle must be rejected by the
 // provider (MemoryProvider treats an empty VolumeHandle as "skip the check",
 // so only a mismatched non-empty value proves the real value is forwarded).
-func TestSnapshotRunningVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
+func TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-running001", 8)
 
-	err := svc.snapshotRunningVolume(doc.VolumeID, "snap-running001", testAccountID)
+	err := svc.snapshotVolume(doc.VolumeID, "snap-running001", testAccountID)
 	require.NoError(t, err, "the correct provider handle must let CreateSnapshot succeed")
 
 	doc.ProviderHandle = "wrong-handle"
 	require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc))
 
-	err = svc.snapshotRunningVolume(doc.VolumeID, "snap-running002", testAccountID)
+	err = svc.snapshotVolume(doc.VolumeID, "snap-running002", testAccountID)
 	require.Error(t, err, "a mismatched provider handle must be rejected, proving the value came from the document")
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
@@ -222,3 +226,98 @@ func TestCreateImageFromInstance_SetsEnaSupportFromInstance(t *testing.T) {
 	require.NotNil(t, ami.EnaSupport)
 	assert.True(t, *ami.EnaSupport)
 }
+
+// createUnavailableProvider refuses every CreateSnapshot as retry-later.
+type createUnavailableProvider struct {
+	ebsprovider.EBSProvider
+}
+
+func (p *createUnavailableProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}
+}
+
+// TestSnapshotVolume_Provider_UnavailableIsRetryLater locks that the
+// CreateImage snapshot surfaces a provider's retry-later as ServiceUnavailable.
+func TestSnapshotVolume_Provider_UnavailableIsRetryLater(t *testing.T) {
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	doc := seedProviderVolume(t, svc, provider, "vol-stopped-busy", 8)
+	svc.SetEBSProvider(&createUnavailableProvider{EBSProvider: provider})
+
+	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped-busy", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServiceUnavailable, err.Error())
+}
+
+// createInUseProvider refuses every CreateSnapshot as volume in use.
+type createInUseProvider struct {
+	ebsprovider.EBSProvider
+}
+
+func (p *createInUseProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "published"}
+}
+
+// TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState locks that the
+// CreateImage snapshot tells the caller to detach a volume the provider refuses as in use.
+func TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState(t *testing.T) {
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	doc := seedProviderVolume(t, svc, provider, "vol-attached-inuse", 8)
+	svc.SetEBSProvider(&createInUseProvider{EBSProvider: provider})
+
+	err := svc.snapshotVolume(doc.VolumeID, "snap-attached-inuse", testAccountID)
+
+	code, message, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "unregistered error: %v", err)
+	assert.Equal(t, awserrors.ErrorIncorrectState, code)
+	assert.Contains(t, message, doc.VolumeID)
+	assert.Contains(t, message, "Detach")
+}
+
+// countingSnapshotProvider counts the CreateSnapshot calls it forwards.
+type countingSnapshotProvider struct {
+	ebsprovider.EBSProvider
+
+	creates atomic.Int32
+}
+
+func (p *countingSnapshotProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	p.creates.Add(1)
+	return p.EBSProvider.CreateSnapshot(ctx, req)
+}
+
+// An image of a running instance is snapshotted by the provider, which drains
+// a buffered volume itself, so CreateImage neither dials the drain socket nor
+// asks the instance's host to drain.
+func TestCreateImageFromInstance_RunningInstanceLeavesTheDrainToTheProvider(t *testing.T) {
+	const instanceID = "i-running-drain"
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	_, nc := testutil.StartTestNATS(t)
+	svc.natsConn = nc
+	svc.config.DataDir = testutil.SocketTempDir(t)
+	counting := &countingSnapshotProvider{EBSProvider: provider}
+	svc.SetEBSProvider(counting)
+
+	doc := seedProviderVolume(t, svc, provider, "vol-running-drain", 8)
+	doc.State, doc.AttachedInstance = "in-use", instanceID
+	require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc))
+	drains := testutil.StartDrainSocket(t, svc.config.DataDir, doc.VolumeID, "OK\n")
+	var commands atomic.Int32
+	sub, err := nc.Subscribe(ec2v1.InstanceCommandSubject(instanceID), func(msg *nats.Msg) {
+		commands.Add(1)
+		_ = msg.Respond([]byte(`{"Code":"InternalError","Message":"no drain socket"}`))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	out, err := svc.CreateImageFromInstance(CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String(instanceID), Name: aws.String("running-ami")},
+		RootVolumeID: doc.VolumeID,
+		IsRunning:    true,
+	}, testAccountID)
+	require.NoError(t, err)
+	require.NotNil(t, out.ImageId)
+
+	assert.Equal(t, int32(1), counting.creates.Load(), "the provider must take the snapshot")
+	assert.Zero(t, drains(), "CreateImage must not dial the drain socket")
+	assert.Zero(t, commands.Load(), "CreateImage must not ask the instance's host to drain")
+}

spinifex/domains/ec2/image/service_impl.go

@@ -38,7 +38,7 @@ type CreateImageParams struct {
 	Input         *ec2.CreateImageInput
 	RootVolumeID  string
 	SourceImageID string
-	IsRunning     bool // true = use live checkpoint (instance still running), false = use numbered checkpoint from Close
+	IsRunning     bool // logged only: the provider's CreateSnapshot drains a live volume itself
 }
 
 // ImageServiceImpl handles AMI image operations with S3 storage.
@@ -69,9 +69,7 @@ func (s *ImageServiceImpl) MetadataStore() *ebsmetadata.Store {
 	return s.metadata
 }
 
-// NewImageServiceImpl creates a new daemon-side image service. natsConn is used
-// to drain a running instance's volume (routed to the node hosting it) before
-// CreateImageFromInstance reads its live checkpoint.
+// NewImageServiceImpl creates a new daemon-side image service.
 func NewImageServiceImpl(cfg *config.Config, natsConn *nats.Conn) *ImageServiceImpl {
 	store := objectstore.NewS3ObjectStoreFromConfig(
 		cfg.Predastore.Host,
@@ -99,8 +97,7 @@ func NewImageServiceImplWithStore(store objectstore.ObjectStore, bucketName stri
 }
 
 // NewImageServiceImplWithConfig creates an image service with an explicit
-// config and NATS connection (for testing the running-volume drain path,
-// which needs config.DataDir/Predastore and a NATS connection to route to).
+// config, object store and NATS connection (for testing).
 func NewImageServiceImplWithConfig(cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn) *ImageServiceImpl {
 	return &ImageServiceImpl{
 		config:     cfg,
@@ -447,15 +444,9 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		"rootVolumeId", params.RootVolumeID, "amiId", amiID, "snapshotId", snapshotID,
 		"isRunning", params.IsRunning)
 
-	// Step 1: Snapshot root volume (live via NATS or offline from S3)
-	var snapshotErr error
-	if params.IsRunning {
-		snapshotErr = s.snapshotRunningVolume(params.RootVolumeID, snapshotID, accountID)
-	} else {
-		snapshotErr = s.snapshotStoppedVolume(params.RootVolumeID, snapshotID, accountID)
-	}
-	if snapshotErr != nil {
-		return nil, snapshotErr
+	// Step 1: Snapshot root volume
+	if err := s.snapshotVolume(params.RootVolumeID, snapshotID, accountID); err != nil {
+		return nil, err
 	}
 
 	// Step 2: Read source volume config for size
@@ -521,51 +512,21 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 	}, nil
 }
 
-// snapshotRunningVolume creates a snapshot of a running instance's volume by
-// draining it to the node hosting it (so any writes buffered there reach S3),
-// then reading the live checkpoint written by nbdkit on every NBD Flush. This
-// shares domains/ec2/snapshot's DrainVolume with ec2.CreateSnapshot so the two
-// live-snapshot paths cannot diverge on when a drain is required.
-func (s *ImageServiceImpl) snapshotRunningVolume(volumeID, snapshotID, accountID string) error {
+// snapshotVolume snapshots the root volume through the EBS provider, whether
+// or not the instance is running. A provider that buffers writes drains the
+// volume inside its own CreateSnapshot, so no drain is issued here.
+func (s *ImageServiceImpl) snapshotVolume(volumeID, snapshotID, accountID string) error {
 	if s.config == nil {
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
 	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, volumeID)
 	if err != nil {
-		slog.Error("snapshotRunningVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
+		slog.Error("snapshotVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// Flush the writes still buffered by whichever node serves the volume, so
-	// the snapshot below is current rather than an arbitrarily stale one
-	// predating e.g. a guest-triggered GPT rewrite (growpart on first boot). An
-	// attached volume that cannot be drained fails here rather than silently
-	// producing an image from a stale checkpoint.
-	if err := ec2snapshot.DrainVolume(context.Background(), s.config, s.store, s.natsConn, volumeID, volMeta.State, volMeta.AttachedInstance, accountID); err != nil {
-		slog.Error("snapshotRunningVolume: drain failed", "volumeId", volumeID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotRunningVolume")
-}
-
-// snapshotStoppedVolume creates a snapshot of a detached volume. No drain: the
-// volume is stopped/detached, so nothing is buffered on a serving node.
-// viperblockd's handleCreateSnapshot prefers a live mounted engine over opening
-// a second one, so it handles the mounted case itself.
-func (s *ImageServiceImpl) snapshotStoppedVolume(volumeID, snapshotID, accountID string) error {
-	if s.config == nil {
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, volumeID)
-	if err != nil {
-		slog.Error("snapshotStoppedVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotStoppedVolume")
+	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotVolume")
 }
 
 // createSnapshot delegates snapshot creation for volMeta to the EBS provider.
@@ -580,7 +541,7 @@ func (s *ImageServiceImpl) createSnapshot(ctx context.Context, volMeta ebsmetada
 	})
 	if err != nil || created == nil {
 		slog.ErrorContext(ctx, "provider CreateSnapshot failed", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
+		return ec2snapshot.ProviderCreateError(volMeta.VolumeID, err)
 	}
 	slog.InfoContext(ctx, "snapshot created", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID)
 	return nil

spinifex/domains/ec2/image/service_impl_test.go

@@ -2551,56 +2551,19 @@ func TestResetImageAttribute_MissingParameters(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorMissingParameter, err.Error())
 }
 
-func TestSnapshotRunningVolume_VolumeConfigMissing(t *testing.T) {
+func TestSnapshotVolume_VolumeConfigMissing(t *testing.T) {
 	store := objectstore.NewMemoryObjectStore()
 	svc := &ImageServiceImpl{
 		config:     &config.Config{},
 		store:      store,
 		bucketName: testBucket,
 	}
-	err := svc.snapshotRunningVolume("vol-missing", "snap-r1", testAccountID)
+	err := svc.snapshotVolume("vol-missing", "snap-s1", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
-func TestSnapshotRunningVolume_BackendInitFails(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:   srv.URL,
-				Region: "us-east-1",
-				Bucket: testBucket,
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-running1", 10)
-
-	err := svc.snapshotRunningVolume("vol-running1", "snap-r2", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotStoppedVolume_VolumeConfigMissing(t *testing.T) {
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config:     &config.Config{},
-		store:      store,
-		bucketName: testBucket,
-	}
-	err := svc.snapshotStoppedVolume("vol-missing", "snap-s1", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotStoppedVolume_BackendInitFails(t *testing.T) {
+func TestSnapshotVolume_BackendInitFails(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusForbidden)
 	}))
@@ -2620,7 +2583,7 @@ func TestSnapshotStoppedVolume_BackendInitFails(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped1", 10)
 
-	err := svc.snapshotStoppedVolume("vol-stopped1", "snap-s2", testAccountID)
+	err := svc.snapshotVolume("vol-stopped1", "snap-s2", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
@@ -2646,60 +2609,7 @@ func imageS3MockServer(t *testing.T) *httptest.Server {
 	}))
 }
 
-func TestSnapshotRunningVolume_EncryptionKeyLoadError(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-	}))
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:   srv.URL,
-				Region: "us-east-1",
-				Bucket: testBucket,
-			},
-			Viperblock: config.ViperblockConfig{
-				EncryptionKeyFile: "/nonexistent-image-run-key.bin",
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-running-enckey", 10)
-
-	err := svc.snapshotRunningVolume("vol-running-enckey", "snap-run-enckey", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotRunningVolume_LoadStateFails(t *testing.T) {
-	srv := imageS3MockServer(t)
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:      srv.URL,
-				Region:    "us-east-1",
-				Bucket:    testBucket,
-				AccessKey: "test-key",
-				SecretKey: "test-secret",
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-running-lsf", 10)
-
-	err := svc.snapshotRunningVolume("vol-running-lsf", "snap-run-lsf", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotStoppedVolume_EncryptionKeyLoadError(t *testing.T) {
+func TestSnapshotVolume_EncryptionKeyLoadError(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 	}))
@@ -2722,12 +2632,12 @@ func TestSnapshotStoppedVolume_EncryptionKeyLoadError(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped-enckey", 10)
 
-	err := svc.snapshotStoppedVolume("vol-stopped-enckey", "snap-stop-enckey", testAccountID)
+	err := svc.snapshotVolume("vol-stopped-enckey", "snap-stop-enckey", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
-func TestSnapshotStoppedVolume_LoadStateFails(t *testing.T) {
+func TestSnapshotVolume_LoadStateFails(t *testing.T) {
 	srv := imageS3MockServer(t)
 	defer srv.Close()
 
@@ -2747,7 +2657,7 @@ func TestSnapshotStoppedVolume_LoadStateFails(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped-lsf", 10)
 
-	err := svc.snapshotStoppedVolume("vol-stopped-lsf", "snap-stop-lsf", testAccountID)
+	err := svc.snapshotVolume("vol-stopped-lsf", "snap-stop-lsf", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }

spinifex/domains/ec2/instance/service_impl.go

@@ -2700,6 +2700,12 @@ func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instanc
 	// failure of this launch.
 	instance.FirstLaunch = false
 
+	if err := s.reconcileStartVolumes(ctx, instance); err != nil {
+		slog.ErrorContext(ctx, "launchClaimedInstance: volumes cannot be reconciled", "instanceId", instance.ID, "err", err)
+		undo()
+		return err
+	}
+
 	instanceType, ok := s.resourceMgr.InstanceTypes()[instance.InstanceType]
 	if !ok {
 		slog.ErrorContext(ctx, "launchClaimedInstance: instance type not available on this node",
@@ -3674,3 +3680,153 @@ func (s *InstanceServiceImpl) DescribeInstanceStatus(ctx context.Context, input
 	slog.InfoContext(ctx, "DescribeInstanceStatus completed", "count", len(statuses))
 	return &ec2.DescribeInstanceStatusOutput{InstanceStatuses: statuses}, nil
 }
+
+// DetachVolumeFromStopped removes a data volume from a stopped instance's
+// record, so a later start no longer attaches it. It does not touch the
+// volume's own state; the caller releases that once the record is clean. The
+// device the volume held comes back on the attachment.
+func (s *InstanceServiceImpl) DetachVolumeFromStopped(ctx context.Context, input *ec2.DetachVolumeInput, accountID string) (*ec2.VolumeAttachment, error) {
+	if input == nil || aws.StringValue(input.InstanceId) == "" || aws.StringValue(input.VolumeId) == "" {
+		return nil, errors.New(awserrors.ErrorMissingParameter)
+	}
+	if s.stoppedStore == nil {
+		slog.ErrorContext(ctx, "DetachVolumeFromStopped: stopped store not available")
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	instanceID, volumeID := *input.InstanceId, *input.VolumeId
+
+	stored, err := s.stoppedStore.LoadStoppedInstance(instanceID)
+	if err != nil {
+		slog.ErrorContext(ctx, "DetachVolumeFromStopped: failed to load stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if stored == nil || !IsInstanceVisible(accountID, stored.AccountID) {
+		return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+	}
+
+	req, ok := findVolumeRequest(stored, volumeID)
+	if !ok {
+		return nil, errors.New(awserrors.ErrorIncorrectState)
+	}
+	if req.EFI {
+		return nil, errors.New(awserrors.ErrorOperationNotPermitted)
+	}
+
+	var removed bool
+	_, err = s.stoppedStore.UpdateStoppedInstance(instanceID, func(v *vm.VM) {
+		removed = removeVolumeFromRecord(v, volumeID)
+	})
+	if err != nil {
+		if errors.Is(err, kvstore.ErrNotFound) {
+			slog.WarnContext(ctx, "DetachVolumeFromStopped: instance claimed concurrently", "instanceId", instanceID)
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		}
+		slog.ErrorContext(ctx, "DetachVolumeFromStopped: failed to update stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if !removed {
+		return nil, errors.New(awserrors.ErrorIncorrectState)
+	}
+
+	return &ec2.VolumeAttachment{
+		VolumeId:   aws.String(volumeID),
+		InstanceId: aws.String(instanceID),
+		Device:     aws.String(req.DeviceName),
+		State:      aws.String(ec2.VolumeAttachmentStateDetaching),
+	}, nil
+}
+
+// AttachVolumeToStopped adds an available volume to a stopped instance's
+// record, so the next start mounts it. The volume is claimed in-use first and
+// released again if the record edit fails, so a failure never leaves the record
+// naming a volume another instance could take.
+func (s *InstanceServiceImpl) AttachVolumeToStopped(ctx context.Context, input *ec2.AttachVolumeInput, accountID string) (*ec2.VolumeAttachment, error) {
+	if input == nil || aws.StringValue(input.InstanceId) == "" || aws.StringValue(input.VolumeId) == "" {
+		return nil, errors.New(awserrors.ErrorMissingParameter)
+	}
+	if s.stoppedStore == nil || s.metadata == nil {
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: stores not available")
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	instanceID, volumeID := *input.InstanceId, *input.VolumeId
+
+	stored, err := s.stoppedStore.LoadStoppedInstance(instanceID)
+	if err != nil {
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to load stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if stored == nil || !IsInstanceVisible(accountID, stored.AccountID) {
+		return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+	}
+
+	meta, err := s.metadata.GetVolume(ctx, accountID, volumeID)
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
+		}
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to read volume", "volumeId", volumeID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if meta.TenantID != accountID {
+		return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
+	}
+	if meta.State != "available" {
+		return nil, errors.New(awserrors.ErrorVolumeInUse)
+	}
+	if meta.AvailabilityZone != "" && s.config != nil && s.config.AZ != "" && meta.AvailabilityZone != s.config.AZ {
+		return nil, errors.New(awserrors.ErrorInvalidVolumeZoneMismatch)
+	}
+
+	device, boot, err := stoppedAttachDevice(stored, aws.StringValue(input.Device))
+	if err != nil {
+		return nil, err
+	}
+
+	meta.State = "in-use"
+	meta.AttachedInstance = instanceID
+	meta.DeviceName = device
+	meta.AttachedAt = time.Now()
+	if err := s.metadata.PutVolume(ctx, meta); err != nil {
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to claim volume", "volumeId", volumeID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
+	var conflict error
+	_, err = s.stoppedStore.UpdateStoppedInstance(instanceID, func(v *vm.VM) {
+		// The device was chosen against a snapshot, so it is checked again
+		// against the record the edit is applied to.
+		if _, _, cerr := stoppedAttachDevice(v, device); cerr != nil {
+			conflict = cerr
+			return
+		}
+		addVolumeToRecord(v, volumeID, device, boot)
+	})
+	if err == nil && conflict != nil {
+		err = conflict
+	}
+	if err != nil {
+		meta.State = "available"
+		meta.AttachedInstance = ""
+		meta.DeviceName = ""
+		meta.AttachedAt = time.Time{}
+		if rerr := s.metadata.PutVolume(ctx, meta); rerr != nil {
+			slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to release volume after a failed record edit", "volumeId", volumeID, "err", rerr)
+		}
+		if conflict != nil {
+			return nil, conflict
+		}
+		if errors.Is(err, kvstore.ErrNotFound) {
+			slog.WarnContext(ctx, "AttachVolumeToStopped: instance claimed concurrently", "instanceId", instanceID)
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		}
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to update stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
+	return &ec2.VolumeAttachment{
+		VolumeId:   aws.String(volumeID),
+		InstanceId: aws.String(instanceID),
+		Device:     aws.String(device),
+		State:      aws.String(ec2.VolumeAttachmentStateAttaching),
+	}, nil
+}

spinifex/domains/ec2/instance/service_nats.go

@@ -1,80 +0,0 @@
-package instance
-
-import (
-	"context"
-	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/nats-io/nats.go"
-)
-
-// NATSInstanceService handles instance operations via NATS. Fan-out methods
-// (DescribeInstances, DescribeInstanceTypes) intentionally error — use the
-// gateway's scatter-gather helper for cluster-wide results.
-type NATSInstanceService struct {
-	natsConn *nats.Conn
-}
-
-var _ InstanceService = (*NATSInstanceService)(nil)
-
-// NewNATSInstanceService creates a new NATS-based instance service.
-func NewNATSInstanceService(conn *nats.Conn) InstanceService {
-	return &NATSInstanceService{natsConn: conn}
-}
-
-func (s *NATSInstanceService) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
-	if input == nil || input.InstanceType == nil {
-		return nil, fmt.Errorf("instance type is required")
-	}
-	topic := fmt.Sprintf("ec2.RunInstances.%s", aws.StringValue(input.InstanceType))
-	return natsmsg.NATSRequest[ec2.Reservation](ctx, s.natsConn, topic, input, 5*time.Minute, accountID)
-}
-
-// DescribeInstances is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstances(_ context.Context, _ *ec2.DescribeInstancesInput, _ string) (*ec2.DescribeInstancesOutput, error) {
-	return nil, fmt.Errorf("DescribeInstances is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-// DescribeInstanceTypes is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstanceTypes(_ context.Context, _ *ec2.DescribeInstanceTypesInput, _ string) (*DescribeInstanceTypesNodeOutput, error) {
-	return nil, fmt.Errorf("DescribeInstanceTypes is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-// DescribeInstanceStatus is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstanceStatus(_ context.Context, _ *ec2.DescribeInstanceStatusInput, _ string) (*ec2.DescribeInstanceStatusOutput, error) {
-	return nil, fmt.Errorf("DescribeInstanceStatus is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-func (s *NATSInstanceService) DescribeInstanceAttribute(ctx context.Context, input *ec2.DescribeInstanceAttributeInput, accountID string) (*ec2.DescribeInstanceAttributeOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstanceAttributeOutput](ctx, s.natsConn, "ec2.DescribeInstanceAttribute", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) DescribeStoppedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstancesOutput](ctx, s.natsConn, "ec2.DescribeStoppedInstances", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) DescribeTerminatedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstancesOutput](ctx, s.natsConn, "ec2.DescribeTerminatedInstances", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) ModifyInstanceAttribute(ctx context.Context, input *ec2.ModifyInstanceAttributeInput, accountID string) (*ec2.ModifyInstanceAttributeOutput, error) {
-	return natsmsg.NATSRequest[ec2.ModifyInstanceAttributeOutput](ctx, s.natsConn, "ec2.ModifyInstanceAttribute", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) ModifyInstanceMetadataOptions(ctx context.Context, input *ec2.ModifyInstanceMetadataOptionsInput, accountID string) (*ec2.ModifyInstanceMetadataOptionsOutput, error) {
-	return natsmsg.NATSRequest[ec2.ModifyInstanceMetadataOptionsOutput](ctx, s.natsConn, "ec2.ModifyInstanceMetadataOptions", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) StartStoppedInstance(ctx context.Context, input *StartStoppedInstanceInput, accountID string) (*StartStoppedInstanceOutput, error) {
-	return natsmsg.NATSRequest[StartStoppedInstanceOutput](ctx, s.natsConn, "ec2.StartStoppedInstance", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) TerminateStoppedInstance(ctx context.Context, input *TerminateStoppedInstanceInput, accountID string) (*TerminateStoppedInstanceOutput, error) {
-	return natsmsg.NATSRequest[TerminateStoppedInstanceOutput](ctx, s.natsConn, "ec2.TerminateStoppedInstance", input, 30*time.Second, accountID)
-}

spinifex/domains/ec2/instance/service_nats_test.go

@@ -1,364 +0,0 @@
-package instance
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/nats-io/nats-server/v2/server"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// startTestNATSServer starts an embedded NATS server for testing.
-func startTestNATSServer(t *testing.T) (*server.Server, string) {
-	t.Helper()
-	ns, nc := testutil.StartTestNATS(t)
-	_ = nc // connection managed by testutil cleanup
-	return ns, ns.ClientURL()
-}
-
-// createValidRunInstancesInput creates a valid RunInstancesInput for testing.
-func createValidRunInstancesInput() *ec2.RunInstancesInput {
-	return &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String("t3.micro"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-		KeyName:      aws.String("test-key-pair"),
-		SecurityGroupIds: []*string{
-			aws.String("sg-0123456789abcdef0"),
-		},
-		SubnetId: aws.String("subnet-6e7f829e"),
-		UserData: aws.String("#!/bin/bash\necho 'test'"),
-	}
-}
-
-// createValidReservation creates a valid ec2.Reservation response for testing.
-func createValidReservation() *ec2.Reservation {
-	return &ec2.Reservation{
-		ReservationId: aws.String("r-0123456789abcdef0"),
-		OwnerId:       aws.String("123456789012"),
-		Instances: []*ec2.Instance{
-			{
-				InstanceId:   aws.String("i-0123456789abcdef0"),
-				InstanceType: aws.String("t3.micro"),
-				ImageId:      aws.String("ami-0abcdef1234567890"),
-				State: &ec2.InstanceState{
-					Code: aws.Int64(0),
-					Name: aws.String("pending"),
-				},
-				PrivateIpAddress: aws.String("10.0.1.100"),
-				SubnetId:         aws.String("subnet-6e7f829e"),
-			},
-		},
-	}
-}
-
-// TestNATSInstanceService_RunInstances_Success tests successful RunInstances operation.
-func TestNATSInstanceService_RunInstances_Success(t *testing.T) {
-	// Skip if LOG_IGNORE is set
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	// Start test NATS server
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	// Create NATS connections
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err, "Failed to connect to NATS")
-	defer nc.Close()
-
-	// Create mock daemon subscriber that responds with valid reservation
-	// Subscribe to the per-instance-type topic (t3.micro from createValidRunInstancesInput)
-	mockReservation := createValidReservation()
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		// Validate that request is properly formatted
-		var input ec2.RunInstancesInput
-		err := json.Unmarshal(msg.Data, &input)
-		if err != nil {
-			t.Errorf("Mock daemon received invalid JSON: %v", err)
-			return
-		}
-
-		// Send back valid reservation
-		responseData, _ := json.Marshal(mockReservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err, "Failed to subscribe mock daemon")
-
-	// Create NATSInstanceService
-	service := NewNATSInstanceService(nc)
-	require.NotNil(t, service)
-
-	// Create valid input
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify success
-	require.NoError(t, err, "RunInstances should succeed")
-	require.NotNil(t, reservation, "Reservation should not be nil")
-
-	// Verify reservation contents
-	assert.Equal(t, *mockReservation.ReservationId, *reservation.ReservationId)
-	assert.Equal(t, *mockReservation.OwnerId, *reservation.OwnerId)
-	require.Len(t, reservation.Instances, 1)
-
-	instance := reservation.Instances[0]
-	assert.Equal(t, "i-0123456789abcdef0", *instance.InstanceId)
-	assert.Equal(t, "t3.micro", *instance.InstanceType)
-	assert.Equal(t, int64(0), *instance.State.Code)
-	assert.Equal(t, "pending", *instance.State.Name)
-}
-
-// TestNATSInstanceService_RunInstances_DaemonError tests error handling when daemon returns error.
-func TestNATSInstanceService_RunInstances_DaemonError(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that responds with error
-	// Subscribe to the per-instance-type topic for the requested type
-	_, err = nc.QueueSubscribe("ec2.RunInstances.invalid.type", "spinifex-workers", func(msg *nats.Msg) {
-		// Send back error response
-		errorResponse := awserrors.GenerateErrorPayload("InvalidInstanceType")
-		msg.Respond(errorResponse)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-
-	input := &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String("invalid.type"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-	}
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify error handling - error should be just the AWS error code for gateway lookup
-	require.Error(t, err, "Should return error when daemon returns error")
-	assert.Nil(t, reservation, "Reservation should be nil on error")
-	assert.Equal(t, "InvalidInstanceType", err.Error(), "Error should be the AWS error code")
-}
-
-// TestNATSInstanceService_RunInstances_NoSubscriber tests behavior when no daemon is subscribed.
-func TestNATSInstanceService_RunInstances_NoSubscriber(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// No subscriber - no daemon running
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	start := time.Now()
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-	duration := time.Since(start)
-
-	// Verify error behavior
-	// When no subscribers are available, NATS returns ErrNoResponders immediately
-	require.Error(t, err, "Should fail when no subscriber")
-	assert.Nil(t, reservation, "Reservation should be nil")
-	assert.ErrorIs(t, err, nats.ErrNoResponders, "Error should be ErrNoResponders")
-
-	// Should fail quickly (within 1 second), not wait for 30 second timeout
-	assert.Less(t, duration.Seconds(), 1.0, "Should fail immediately when no responders")
-}
-
-// TestNATSInstanceService_RunInstances_InvalidResponse tests handling of malformed responses.
-func TestNATSInstanceService_RunInstances_InvalidResponse(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that responds with invalid JSON
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		// Send back malformed JSON
-		msg.Respond([]byte(`{"invalid": json response`))
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify error handling
-	require.Error(t, err, "Should return error for invalid JSON")
-	assert.Nil(t, reservation, "Reservation should be nil")
-	assert.Contains(t, err.Error(), "failed to unmarshal", "Error should indicate unmarshal failure")
-}
-
-// TestNATSInstanceService_RunInstances_MarshalError tests handling when input marshaling fails.
-func TestNATSInstanceService_RunInstances_MarshalError(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	service := NewNATSInstanceService(nc)
-
-	// Nil input should be rejected before hitting NATS
-	reservation, err := service.RunInstances(context.Background(), nil, "123456789012")
-	require.Error(t, err, "Should handle nil input")
-	assert.Nil(t, reservation)
-	assert.Contains(t, err.Error(), "instance type is required")
-}
-
-// TestNATSInstanceService_RunInstances_MultipleInstances tests launching multiple instances.
-func TestNATSInstanceService_RunInstances_MultipleInstances(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that returns multiple instances
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		var input ec2.RunInstancesInput
-		json.Unmarshal(msg.Data, &input)
-
-		// Create reservation with count matching MinCount
-		count := int(*input.MinCount)
-		instances := make([]*ec2.Instance, count)
-		for i := range count {
-			instanceId := aws.String(aws.StringValue(aws.String(fmt.Sprintf("i-%d", i))))
-			instances[i] = &ec2.Instance{
-				InstanceId:   instanceId,
-				InstanceType: input.InstanceType,
-				ImageId:      input.ImageId,
-				State: &ec2.InstanceState{
-					Code: aws.Int64(0),
-					Name: aws.String("pending"),
-				},
-			}
-		}
-
-		reservation := &ec2.Reservation{
-			ReservationId: aws.String("r-0123456789abcdef0"),
-			Instances:     instances,
-		}
-
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-
-	// Request 3 instances
-	input := createValidRunInstancesInput()
-	input.MinCount = aws.Int64(3)
-	input.MaxCount = aws.Int64(3)
-
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify multiple instances returned
-	require.NoError(t, err)
-	require.NotNil(t, reservation)
-	assert.Len(t, reservation.Instances, 3, "Should have 3 instances")
-}
-
-// TestNATSInstanceService_RunInstances_QueueGroup tests load balancing with queue groups.
-func TestNATSInstanceService_RunInstances_QueueGroup(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create multiple mock daemon workers in same queue group
-	worker1Count := 0
-	worker2Count := 0
-
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		worker1Count++
-		reservation := createValidReservation()
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		worker2Count++
-		reservation := createValidReservation()
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Send multiple requests
-	requestCount := 10
-	for range requestCount {
-		reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-		require.NoError(t, err)
-		require.NotNil(t, reservation)
-	}
-
-	// Verify load was distributed (both workers should have handled some requests)
-	// With queue groups, NATS distributes messages across subscribers
-	t.Logf("Worker 1 handled: %d requests", worker1Count)
-	t.Logf("Worker 2 handled: %d requests", worker2Count)
-
-	assert.Positive(t, worker1Count, "Worker 1 should handle some requests")
-	assert.Positive(t, worker2Count, "Worker 2 should handle some requests")
-	assert.Equal(t, requestCount, worker1Count+worker2Count, "All requests should be handled")
-}

spinifex/domains/ec2/instance/start_after_detach_test.go

@@ -0,0 +1,54 @@
+//test:in-package — it swaps the manager's volume mounter, an unexported dependency.
+package instance
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// mountRecorder notes which volumes a launch asked to mount and then fails the
+// launch, which is the earliest point that shows what the start would attach.
+type mountRecorder struct {
+	refusingMounter
+
+	mu      sync.Mutex
+	mounted []string
+}
+
+func (r *mountRecorder) Mount(_ context.Context, v *vm.VM) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, req := range v.EBSRequests.Requests {
+		r.mounted = append(r.mounted, req.Name)
+	}
+	return errors.New("stop after recording")
+}
+
+func (r *mountRecorder) MountOne(context.Context, string, *viperblocklegacyv1.EBSRequest) error {
+	return nil
+}
+
+// A volume detached while the instance was stopped must not come back when the
+// instance is started: the start path reads the same record the detach edited.
+func TestStartStoppedInstance_DoesNotReattachAVolumeDetachedWhileStopped(t *testing.T) {
+	id := "i-detach-then-start"
+	stopped := stoppedWithVolumes("acc", id)
+	stopped.InstanceType = "t3.micro"
+	f := newStartFixture(t, stopped)
+	rec := &mountRecorder{}
+	f.svc.vmMgr.SetDeps(vm.Deps{VolumeMounter: rec})
+
+	_, err := f.svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-data"), "acc")
+	require.NoError(t, err)
+	require.Error(t, f.start(t, id), "the recorder fails the launch after the mount")
+
+	assert.Equal(t, []string{"vol-root", "vol-other"}, rec.mounted,
+		"the detached volume is not part of the start")
+}

spinifex/domains/ec2/instance/start_volumes.go

@@ -0,0 +1,88 @@
+package instance
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"slices"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+)
+
+// reconcileStartVolumes drops the data volumes a claimed instance can no longer
+// have, and fails the start when its boot volume is gone.
+func (s *InstanceServiceImpl) reconcileStartVolumes(ctx context.Context, instance *vm.VM) error {
+	// A direct-boot system guest boots from a kernel, not a root volume, and its
+	// one data volume, if any, is its own store rather than a customer's.
+	if s.metadata == nil || instance.DirectBoot {
+		return nil
+	}
+
+	instance.EBSRequests.Mu.Lock()
+	requests := slices.Clone(instance.EBSRequests.Requests)
+	instance.EBSRequests.Mu.Unlock()
+
+	if !slices.ContainsFunc(requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Boot }) {
+		return noRootVolumeError(instance)
+	}
+
+	var drop []string
+	for _, req := range requests {
+		meta, err := s.metadata.GetVolume(ctx, instance.AccountID, req.Name)
+		missing := err != nil && objectstore.IsNoSuchKeyError(err)
+		if err != nil && !missing {
+			return fmt.Errorf("read volume %s: %w", req.Name, err)
+		}
+		if req.Boot {
+			if missing {
+				slog.ErrorContext(ctx, "start: root volume no longer exists", "instanceId", instance.ID, "volumeId", req.Name)
+				return noRootVolumeError(instance)
+			}
+			continue
+		}
+		if req.EFI {
+			continue
+		}
+		if missing {
+			slog.WarnContext(ctx, "start: dropping deleted data volume", "instanceId", instance.ID, "volumeId", req.Name)
+			drop = append(drop, req.Name)
+			continue
+		}
+		if meta.AttachedInstance != "" && meta.AttachedInstance != instance.ID {
+			slog.WarnContext(ctx, "start: dropping data volume attached to another instance",
+				"instanceId", instance.ID, "volumeId", req.Name, "attachedTo", meta.AttachedInstance)
+			drop = append(drop, req.Name)
+		}
+	}
+	if len(drop) == 0 {
+		return nil
+	}
+
+	instance.EBSRequests.Mu.Lock()
+	instance.EBSRequests.Requests = slices.DeleteFunc(instance.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool {
+		return slices.Contains(drop, r.Name)
+	})
+	instance.EBSRequests.Mu.Unlock()
+	if instance.Instance != nil {
+		instance.Instance.BlockDeviceMappings = slices.DeleteFunc(instance.Instance.BlockDeviceMappings, func(m *ec2.InstanceBlockDeviceMapping) bool {
+			return m != nil && m.Ebs != nil && slices.Contains(drop, aws.StringValue(m.Ebs.VolumeId))
+		})
+	}
+	return nil
+}
+
+// noRootVolumeError is what AWS answers when an instance is started with
+// nothing attached at its root device name.
+func noRootVolumeError(instance *vm.VM) error {
+	device := ebsmetadata.RootDeviceName
+	if instance.Instance != nil && aws.StringValue(instance.Instance.RootDeviceName) != "" {
+		device = aws.StringValue(instance.Instance.RootDeviceName)
+	}
+	return awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "Instance does not have a volume attached at root (%s)", device)
+}

spinifex/domains/ec2/instance/start_volumes_test.go

@@ -0,0 +1,142 @@
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const startVolumesAccount = "000000000011"
+
+func startVolumesService(t *testing.T, volumes ...ebsmetadata.Volume) *InstanceServiceImpl {
+	t.Helper()
+	store := objectstore.NewMemoryObjectStore()
+	meta := ebsmetadata.NewStore(store, testRootBucket)
+	for _, v := range volumes {
+		v.TenantID = startVolumesAccount
+		require.NoError(t, meta.PutVolume(context.Background(), v))
+	}
+	return &InstanceServiceImpl{metadata: meta}
+}
+
+func startingInstance(id string, names ...string) *vm.VM {
+	v := &vm.VM{ID: id, AccountID: startVolumesAccount, Status: vm.StateStopped, Instance: &ec2.Instance{InstanceId: aws.String(id)}}
+	for i, n := range names {
+		boot := i == 0
+		device := "/dev/sd" + string(rune('f'+i))
+		if boot {
+			device = "/dev/sda1"
+		}
+		v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{Name: n, Boot: boot, DeviceName: device})
+		v.Instance.BlockDeviceMappings = append(v.Instance.BlockDeviceMappings, &ec2.InstanceBlockDeviceMapping{
+			DeviceName: aws.String(device), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String(n)},
+		})
+	}
+	return v
+}
+
+func requestedNames(v *vm.VM) []string {
+	var out []string
+	for _, r := range v.EBSRequests.Requests {
+		out = append(out, r.Name)
+	}
+	return out
+}
+
+func mappedNames(v *vm.VM) []string {
+	var out []string
+	for _, m := range v.Instance.BlockDeviceMappings {
+		out = append(out, aws.StringValue(m.Ebs.VolumeId))
+	}
+	return out
+}
+
+// An instance stopped on an older build had its data volumes released, and they
+// may since have been attached elsewhere or deleted; starting must not take a
+// volume that is no longer its own.
+func TestReconcileStartVolumes_DropsVolumesThatAreNoLongerTheInstances(t *testing.T) {
+	svc := startVolumesService(t,
+		ebsmetadata.Volume{VolumeID: "vol-root", State: "in-use", AttachedInstance: "i-old"},
+		ebsmetadata.Volume{VolumeID: "vol-elsewhere", State: "in-use", AttachedInstance: "i-someone-else"},
+		ebsmetadata.Volume{VolumeID: "vol-free", State: "available"},
+		ebsmetadata.Volume{VolumeID: "vol-mine", State: "in-use", AttachedInstance: "i-old"},
+	)
+	inst := startingInstance("i-old", "vol-root", "vol-elsewhere", "vol-gone", "vol-free", "vol-mine")
+
+	require.NoError(t, svc.reconcileStartVolumes(context.Background(), inst))
+
+	want := []string{"vol-root", "vol-free", "vol-mine"}
+	assert.Equal(t, want, requestedNames(inst),
+		"attached-elsewhere and deleted volumes are dropped; free and own ones stay")
+	assert.Equal(t, want, mappedNames(inst), "the API view stops listing them too")
+}
+
+func TestReconcileStartVolumes_MissingBootVolumeFailsTheStart(t *testing.T) {
+	svc := startVolumesService(t, ebsmetadata.Volume{VolumeID: "vol-data", State: "available"})
+	inst := startingInstance("i-old", "vol-root", "vol-data")
+	inst.Instance.RootDeviceName = aws.String("/dev/sda1")
+
+	err := svc.reconcileStartVolumes(context.Background(), inst)
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+	assert.Contains(t, err.Error(), "Instance does not have a volume attached at root (/dev/sda1)")
+}
+
+// A root volume detached while the instance was stopped leaves no boot request
+// in the record; AWS refuses the start with the same answer.
+func TestReconcileStartVolumes_NoRootVolumeAttachedFailsTheStart(t *testing.T) {
+	svc := startVolumesService(t, ebsmetadata.Volume{VolumeID: "vol-data", State: "in-use", AttachedInstance: "i-old"})
+	inst := startingInstance("i-old", "vol-root", "vol-data")
+	require.True(t, removeVolumeFromRecord(inst, "vol-root"))
+	inst.Instance.RootDeviceName = aws.String("/dev/sda1")
+
+	err := svc.reconcileStartVolumes(context.Background(), inst)
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+	assert.Contains(t, err.Error(), "Instance does not have a volume attached at root (/dev/sda1)")
+	assert.Equal(t, []string{"vol-data"}, requestedNames(inst), "a refused start changes nothing")
+}
+
+// An instance stopped on an older build had its data volumes released to
+// available with no attachment; those are still the instance's own and a
+// start must bring them back.
+func TestReconcileStartVolumes_ReattachesStillAvailableUnattachedVolume(t *testing.T) {
+	svc := startVolumesService(t,
+		ebsmetadata.Volume{VolumeID: "vol-root", State: "in-use", AttachedInstance: "i-old"},
+		ebsmetadata.Volume{VolumeID: "vol-released", State: "available"},
+	)
+	inst := startingInstance("i-old", "vol-root", "vol-released")
+
+	require.NoError(t, svc.reconcileStartVolumes(context.Background(), inst))
+
+	assert.Equal(t, []string{"vol-root", "vol-released"}, requestedNames(inst))
+	assert.Equal(t, []string{"vol-root", "vol-released"}, mappedNames(inst))
+}
+
+// ALB and SQS guests boot from a kernel with no root volume, so a start or a
+// recovery must not demand one; an SQS store's data volume stays named.
+func TestReconcileStartVolumes_DirectBootNeedsNoRootVolume(t *testing.T) {
+	t.Parallel()
+	svc := startVolumesService(t)
+
+	alb := &vm.VM{ID: "i-alb", AccountID: startVolumesAccount, Status: vm.StateStopped, DirectBoot: true,
+		Instance: &ec2.Instance{InstanceId: aws.String("i-alb")}}
+	require.NoError(t, svc.reconcileStartVolumes(context.Background(), alb))
+
+	sqs := &vm.VM{ID: "i-sqs", AccountID: startVolumesAccount, Status: vm.StateStopped, DirectBoot: true,
+		Instance: &ec2.Instance{InstanceId: aws.String("i-sqs")}}
+	sqs.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-store-gone", DeviceName: "/dev/vdb"}}
+	require.NoError(t, svc.reconcileStartVolumes(context.Background(), sqs))
+	assert.Equal(t, []string{"vol-store-gone"}, requestedNames(sqs), "a system store volume is never pruned")
+}

spinifex/domains/ec2/instance/stopped_attach_test.go

@@ -0,0 +1,180 @@
+//test:in-package — it swaps the manager's volume mounter, an unexported dependency.
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const stoppedAttachAZ = "ap-southeast-2a"
+
+type stoppedAttachFixture struct {
+	svc    *InstanceServiceImpl
+	stored *vm.VM
+	id     string
+}
+
+func newStoppedAttachFixture(t *testing.T, volumes ...ebsmetadata.Volume) *stoppedAttachFixture {
+	t.Helper()
+	id := "i-stopped-attach"
+	svc := startVolumesService(t, volumes...)
+	stored := startingInstance(id, "vol-root", "vol-data")
+	svc.config = &config.Config{AZ: stoppedAttachAZ}
+	svc.vmMgr = mgrWith(map[string]*vm.VM{})
+	svc.stoppedStore = &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	return &stoppedAttachFixture{svc: svc, stored: stored, id: id}
+}
+
+func (f *stoppedAttachFixture) attach(volumeID, device string) (*ec2.VolumeAttachment, error) {
+	in := &ec2.AttachVolumeInput{InstanceId: aws.String(f.id), VolumeId: aws.String(volumeID)}
+	if device != "" {
+		in.Device = aws.String(device)
+	}
+	return f.svc.AttachVolumeToStopped(context.Background(), in, startVolumesAccount)
+}
+
+func (f *stoppedAttachFixture) volume(t *testing.T, id string) ebsmetadata.Volume {
+	t.Helper()
+	v, err := f.svc.metadata.GetVolume(context.Background(), startVolumesAccount, id)
+	require.NoError(t, err)
+	return v
+}
+
+func availableVolume(id string) ebsmetadata.Volume {
+	return ebsmetadata.Volume{VolumeID: id, State: "available", AvailabilityZone: stoppedAttachAZ}
+}
+
+// AWS lets a volume be attached to a stopped instance; the record then names it
+// so the next start mounts it, and the volume is held in-use meanwhile.
+func TestAttachVolumeToStopped_AddsTheVolumeAndClaimsIt(t *testing.T) {
+	f := newStoppedAttachFixture(t, availableVolume("vol-new"))
+
+	out, err := f.attach("vol-new", "")
+	require.NoError(t, err)
+
+	assert.Equal(t, "/dev/sdf", aws.StringValue(out.Device), "the first free data device")
+	assert.Equal(t, []string{"vol-root", "vol-data", "vol-new"}, requestedNames(f.stored))
+	assert.Equal(t, []string{"vol-root", "vol-data", "vol-new"}, mappedNames(f.stored))
+	assert.Equal(t, vm.StateStopped, f.stored.Status)
+	got := f.volume(t, "vol-new")
+	assert.Equal(t, "in-use", got.State)
+	assert.Equal(t, f.id, got.AttachedInstance)
+	assert.Equal(t, "/dev/sdf", got.DeviceName)
+	for _, r := range f.stored.EBSRequests.Requests {
+		if r.Name == "vol-new" {
+			assert.False(t, r.Boot, "a data device is not the root")
+		}
+	}
+}
+
+func TestAttachVolumeToStopped_Refusals(t *testing.T) {
+	tests := []struct {
+		name    string
+		volumes []ebsmetadata.Volume
+		volume  string
+		device  string
+		want    string
+	}{
+		{"volume in use elsewhere", []ebsmetadata.Volume{{VolumeID: "vol-busy", State: "in-use", AttachedInstance: "i-other", AvailabilityZone: stoppedAttachAZ}}, "vol-busy", "", awserrors.ErrorVolumeInUse},
+		{"wrong availability zone", []ebsmetadata.Volume{{VolumeID: "vol-far", State: "available", AvailabilityZone: "ap-southeast-2b"}}, "vol-far", "", awserrors.ErrorInvalidVolumeZoneMismatch},
+		{"no such volume", nil, "vol-missing", "", awserrors.ErrorInvalidVolumeNotFound},
+		{"device already taken", []ebsmetadata.Volume{availableVolume("vol-new")}, "vol-new", "/dev/sdg", awserrors.ErrorInvalidParameterValue},
+		{"root device already has a volume", []ebsmetadata.Volume{availableVolume("vol-new")}, "vol-new", "/dev/sda1", awserrors.ErrorInvalidParameterValue},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := newStoppedAttachFixture(t, tt.volumes...)
+
+			_, err := f.attach(tt.volume, tt.device)
+
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, tt.want), "got %v, want %s", err, tt.want)
+			assert.Equal(t, []string{"vol-root", "vol-data"}, requestedNames(f.stored), "a refused attach changes nothing")
+			for _, v := range tt.volumes {
+				got := f.volume(t, v.VolumeID)
+				assert.Equal(t, v.State, got.State, "a refused attach leaves the volume as it was")
+				assert.Equal(t, v.AttachedInstance, got.AttachedInstance)
+			}
+		})
+	}
+}
+
+func TestAttachVolumeToStopped_OtherAccountsInstanceIsNotFound(t *testing.T) {
+	f := newStoppedAttachFixture(t, availableVolume("vol-new"))
+
+	_, err := f.svc.AttachVolumeToStopped(context.Background(),
+		&ec2.AttachVolumeInput{InstanceId: aws.String(f.id), VolumeId: aws.String("vol-new")}, "000000000099")
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidInstanceIDNotFound))
+	assert.Equal(t, []string{"vol-root", "vol-data"}, requestedNames(f.stored))
+}
+
+// A record edit that cannot be applied must give the volume back, otherwise it
+// stays in-use against an instance that does not name it.
+func TestAttachVolumeToStopped_FailedRecordEditReleasesTheVolume(t *testing.T) {
+	f := newStoppedAttachFixture(t, availableVolume("vol-new"))
+	f.svc.stoppedStore = &vmmock.StateStore{Stopped: map[string]*vm.VM{f.id: f.stored}, UpdateStoppedErr: assert.AnError}
+
+	_, err := f.attach("vol-new", "")
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorServerInternal))
+	got := f.volume(t, "vol-new")
+	assert.Equal(t, "available", got.State)
+	assert.Empty(t, got.AttachedInstance)
+}
+
+// Detaching the root volume of a stopped instance makes it unstartable; the way
+// back is attaching a volume at the root device name, which makes it the boot
+// volume again so the start finds a root.
+func TestAttachVolumeToStopped_RootDeviceNameRestoresTheRootAndTheStartWorks(t *testing.T) {
+	id := "i-root-roundtrip"
+	stopped := stoppedWithVolumes(startVolumesAccount, id)
+	stopped.InstanceType = "t3.micro"
+	stopped.Instance.RootDeviceName = aws.String("/dev/sda1")
+	f := newStartFixture(t, stopped)
+	rec := &mountRecorder{}
+	f.svc.vmMgr.SetDeps(vm.Deps{VolumeMounter: rec})
+	f.svc.config = &config.Config{AZ: stoppedAttachAZ}
+	meta := startVolumesService(t,
+		ebsmetadata.Volume{VolumeID: "vol-root", State: "in-use", AttachedInstance: id, AvailabilityZone: stoppedAttachAZ},
+		ebsmetadata.Volume{VolumeID: "vol-data", State: "in-use", AttachedInstance: id, AvailabilityZone: stoppedAttachAZ},
+		ebsmetadata.Volume{VolumeID: "vol-other", State: "in-use", AttachedInstance: id, AvailabilityZone: stoppedAttachAZ},
+	)
+	f.svc.metadata = meta.metadata
+	stopped.AccountID = startVolumesAccount
+
+	_, err := f.svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-root"), startVolumesAccount)
+	require.NoError(t, err)
+	vol, err := f.svc.metadata.GetVolume(context.Background(), startVolumesAccount, "vol-root")
+	require.NoError(t, err)
+	vol.State, vol.AttachedInstance = "available", ""
+	require.NoError(t, f.svc.metadata.PutVolume(context.Background(), vol))
+
+	out, err := f.svc.AttachVolumeToStopped(context.Background(),
+		&ec2.AttachVolumeInput{InstanceId: aws.String(id), VolumeId: aws.String("vol-root"), Device: aws.String("/dev/sda1")}, startVolumesAccount)
+	require.NoError(t, err)
+	assert.Equal(t, "/dev/sda1", aws.StringValue(out.Device))
+	var boot bool
+	for _, r := range stopped.EBSRequests.Requests {
+		if r.Name == "vol-root" {
+			boot = r.Boot
+		}
+	}
+	assert.True(t, boot, "the root device name makes it the boot volume")
+
+	_, err = f.svc.StartStoppedInstance(context.Background(), &StartStoppedInstanceInput{InstanceID: id}, startVolumesAccount)
+	require.Error(t, err, "the recorder fails the launch after the mount")
+	assert.Contains(t, rec.mounted, "vol-root", "the start now mounts the restored root")
+}

spinifex/domains/ec2/instance/stopped_detach.go

@@ -0,0 +1,112 @@
+package instance
+
+import (
+	"errors"
+	"fmt"
+	"slices"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+)
+
+func findVolumeRequest(v *vm.VM, volumeID string) (viperblocklegacyv1.EBSRequest, bool) {
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	i := slices.IndexFunc(v.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Name == volumeID })
+	if i < 0 {
+		return viperblocklegacyv1.EBSRequest{}, false
+	}
+	return v.EBSRequests.Requests[i], true
+}
+
+// removeVolumeFromRecord drops the volume from the mount list and the API view
+// of the record, and reports whether it was a data volume that was there.
+func removeVolumeFromRecord(v *vm.VM, volumeID string) bool {
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	i := slices.IndexFunc(v.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Name == volumeID })
+	if i < 0 || v.EBSRequests.Requests[i].EFI {
+		return false
+	}
+	v.EBSRequests.Requests = slices.Delete(v.EBSRequests.Requests, i, i+1)
+	if v.Instance != nil {
+		v.Instance.BlockDeviceMappings = slices.DeleteFunc(v.Instance.BlockDeviceMappings, func(m *ec2.InstanceBlockDeviceMapping) bool {
+			return m != nil && m.Ebs != nil && aws.StringValue(m.Ebs.VolumeId) == volumeID
+		})
+	}
+	return true
+}
+
+// stoppedAttachDevice picks the device a volume takes on a stopped instance and
+// reports whether it is the root. An empty request takes the next free
+// /dev/sd[f-p]; a named device must be free, and the root device name is free
+// only while no boot volume is attached.
+func stoppedAttachDevice(v *vm.VM, requested string) (string, bool, error) {
+	root := ebsmetadata.RootDeviceName
+	if v.Instance != nil && aws.StringValue(v.Instance.RootDeviceName) != "" {
+		root = aws.StringValue(v.Instance.RootDeviceName)
+	}
+
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	used := make(map[string]bool, len(v.EBSRequests.Requests))
+	hasBoot := false
+	for _, r := range v.EBSRequests.Requests {
+		if r.DeviceName != "" {
+			used[r.DeviceName] = true
+		}
+		hasBoot = hasBoot || r.Boot
+	}
+	if v.Instance != nil {
+		for _, m := range v.Instance.BlockDeviceMappings {
+			if m != nil && m.DeviceName != nil {
+				used[*m.DeviceName] = true
+			}
+		}
+	}
+
+	if requested == "" {
+		for c := 'f'; c <= 'p'; c++ {
+			if dev := fmt.Sprintf("/dev/sd%c", c); !used[dev] {
+				return dev, false, nil
+			}
+		}
+		return "", false, errors.New(awserrors.ErrorAttachmentLimitExceeded)
+	}
+	if used[requested] {
+		return "", false, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	boot := requested == root
+	if boot && hasBoot {
+		return "", false, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	return requested, boot, nil
+}
+
+// addVolumeToRecord adds the volume to the mount list and the API view of the
+// record. The root device name makes it the boot volume.
+func addVolumeToRecord(v *vm.VM, volumeID, device string, boot bool) {
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
+		Name:       volumeID,
+		DeviceName: device,
+		Boot:       boot,
+	})
+	if v.Instance == nil {
+		return
+	}
+	mapping := &ec2.InstanceBlockDeviceMapping{}
+	mapping.SetDeviceName(device)
+	mapping.Ebs = &ec2.EbsInstanceBlockDevice{}
+	mapping.Ebs.SetVolumeId(volumeID)
+	mapping.Ebs.SetAttachTime(time.Now())
+	mapping.Ebs.SetDeleteOnTermination(false)
+	mapping.Ebs.SetStatus("attached")
+	v.Instance.BlockDeviceMappings = append(v.Instance.BlockDeviceMappings, mapping)
+}

spinifex/domains/ec2/instance/stopped_detach_test.go

@@ -0,0 +1,139 @@
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func stoppedWithVolumes(owner, id string) *vm.VM {
+	v := &vm.VM{
+		ID: id, AccountID: owner, Status: vm.StateStopped,
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(id),
+			BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
+				{DeviceName: aws.String("/dev/sda1"), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-root")}},
+				{DeviceName: aws.String("/dev/sdf"), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-data")}},
+				{DeviceName: aws.String("/dev/sdg"), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-other")}},
+			},
+		},
+	}
+	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+		{Name: "vol-root", Boot: true, DeviceName: "/dev/sda1"},
+		{Name: "vol-data", DeviceName: "/dev/sdf"},
+		{Name: "vol-other", DeviceName: "/dev/sdg"},
+	}
+	return v
+}
+
+func stoppedDetachInput(instanceID, volumeID string) *ec2.DetachVolumeInput {
+	return &ec2.DetachVolumeInput{InstanceId: aws.String(instanceID), VolumeId: aws.String(volumeID)}
+}
+
+func volumeNames(v *vm.VM) []string {
+	var out []string
+	for _, r := range v.EBSRequests.Requests {
+		out = append(out, r.Name)
+	}
+	return out
+}
+
+// AWS lets a stopped instance's data volume be detached; the record must stop
+// claiming it so a later start does not re-attach it.
+func TestDetachVolumeFromStopped_RemovesOnlyThatVolume(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	id := "i-stopped-detach"
+	stored := stoppedWithVolumes(owner, id)
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+	out, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-data"), owner)
+	require.NoError(t, err)
+
+	assert.Equal(t, "/dev/sdf", aws.StringValue(out.Device))
+	assert.Equal(t, id, aws.StringValue(out.InstanceId))
+	assert.Equal(t, []string{"vol-root", "vol-other"}, volumeNames(stored))
+	var mapped []string
+	for _, m := range stored.Instance.BlockDeviceMappings {
+		mapped = append(mapped, aws.StringValue(m.Ebs.VolumeId))
+	}
+	assert.Equal(t, []string{"vol-root", "vol-other"}, mapped)
+	assert.Equal(t, vm.StateStopped, stored.Status, "the instance stays stopped")
+}
+
+func TestDetachVolumeFromStopped_Refusals(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	tests := []struct {
+		name    string
+		caller  string
+		volume  string
+		stopped bool
+		want    string
+	}{
+		{"volume not on the instance", owner, "vol-elsewhere", true, awserrors.ErrorIncorrectState},
+		{"another account", "000000000099", "vol-data", true, awserrors.ErrorInvalidInstanceIDNotFound},
+		{"no such stopped instance", owner, "vol-data", false, awserrors.ErrorInvalidInstanceIDNotFound},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			id := "i-stopped-refuse"
+			stored := stoppedWithVolumes(owner, id)
+			store := &vmmock.StateStore{Stopped: map[string]*vm.VM{}}
+			if tt.stopped {
+				store.Stopped[id] = stored
+			}
+			svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+			_, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, tt.volume), tt.caller)
+
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, tt.want), "got %v, want %s", err, tt.want)
+			assert.Equal(t, []string{"vol-root", "vol-data", "vol-other"}, volumeNames(stored),
+				"a refused detach changes nothing")
+		})
+	}
+}
+
+// AWS lets the root volume of a stopped instance be detached; the instance then
+// cannot start until a volume is attached at its root device name again.
+func TestDetachVolumeFromStopped_DetachesTheRootVolume(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	id := "i-stopped-root"
+	stored := stoppedWithVolumes(owner, id)
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+	out, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-root"), owner)
+	require.NoError(t, err)
+
+	assert.Equal(t, "/dev/sda1", aws.StringValue(out.Device))
+	assert.Equal(t, []string{"vol-data", "vol-other"}, volumeNames(stored))
+	assert.Equal(t, vm.StateStopped, stored.Status)
+}
+
+// The EFI variable store is Spinifex-internal and never shown to the customer,
+// so it cannot be detached even by its name.
+func TestDetachVolumeFromStopped_RefusesTheInternalEFIStore(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	id := "i-stopped-efi"
+	stored := stoppedWithVolumes(owner, id)
+	stored.EBSRequests.Requests = append(stored.EBSRequests.Requests,
+		viperblocklegacyv1.EBSRequest{Name: "vol-efi", EFI: true})
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+	_, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-efi"), owner)
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorOperationNotPermitted), "got %v", err)
+	assert.Equal(t, []string{"vol-root", "vol-data", "vol-other", "vol-efi"}, volumeNames(stored))
+}

spinifex/domains/ec2/instance/uefistore/uefistore.go

@@ -0,0 +1,185 @@
+// Package uefistore holds the types that describe an instance's UEFI variable
+// store and the bucket of store records that carry its deletion and write-back
+// obligations past the life of the instance record.
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+const (
+	// BucketName holds one store record per instance, keyed by instance ID.
+	BucketName = "spinifex-uefi-stores"
+	// BucketVersion is the bucket's schema stamp.
+	BucketVersion = 1
+	// RecordSchemaVersion is the version every record is written at.
+	RecordSchemaVersion uint16 = 1
+
+	// StateLive is a store whose objects belong to a running or stopped instance.
+	StateLive = "live"
+	// StateDeleting is a store whose objects are to be deleted, and then the record.
+	StateDeleting = "deleting"
+
+	objectRoot = "uefi/"
+)
+
+// ErrInvalidRecord is returned for a store record that breaks its invariants,
+// whether it is about to be written or was just read.
+var ErrInvalidRecord = errors.New("uefistore: invalid store record")
+
+// FirmwareIdentity names the firmware a variable store was written under. A
+// store only boots where Arch and VarsSize match the node's firmware, because
+// pflash needs a byte-exact region; CodeSHA256 is recorded, not enforced.
+type FirmwareIdentity struct {
+	Arch       string `json:"arch"`
+	VarsSize   int64  `json:"vars_size"`
+	CodeSHA256 string `json:"code_sha256"`
+}
+
+// UEFIStoreRecord is the durable owner of one instance's variable store
+// objects. It outlives the instance record, so an obligation recorded here is
+// not lost when a terminated record expires.
+type UEFIStoreRecord struct {
+	SchemaVersion uint16         `json:"schema_version"`
+	InstanceID    string         `json:"instance_id"`
+	AccountID     string         `json:"account_id"`
+	State         string         `json:"state"`
+	WriteBack     *UEFIWriteBack `json:"write_back,omitempty"`
+}
+
+// UEFIWriteBack is a write-back that failed and is still owed. Key is the
+// object every retry puts to, so retries leave no extra objects; SeededFrom is
+// the pointer Node seeded from, which a retry must still find on the record.
+type UEFIWriteBack struct {
+	Node       string    `json:"node"`
+	Key        string    `json:"key"`
+	SeededFrom string    `json:"seeded_from"`
+	Since      time.Time `json:"since"`
+}
+
+// KeyPrefix is the object-store prefix every store object of ownerID (an
+// instance or image ID) is written under.
+func KeyPrefix(ownerID string) string {
+	return objectRoot + ownerID + "/"
+}
+
+// ownsKey reports whether key names one object directly under ownerID's prefix.
+func ownsKey(ownerID, key string) bool {
+	name, ok := strings.CutPrefix(key, KeyPrefix(ownerID))
+	return ok && name != "" && !strings.Contains(name, "/")
+}
+
+// validate checks rec against key. A write-back key outside the instance's own
+// prefix is refused because a superseded obligation deletes that key.
+func (rec *UEFIStoreRecord) validate(key string) error {
+	switch {
+	case rec.SchemaVersion > RecordSchemaVersion:
+		return fmt.Errorf("%w: %s is at schema_version %d, newer than this node understands (%d)",
+			ErrInvalidRecord, key, rec.SchemaVersion, RecordSchemaVersion)
+	case rec.SchemaVersion != RecordSchemaVersion:
+		return fmt.Errorf("%w: %s has schema_version %d", ErrInvalidRecord, key, rec.SchemaVersion)
+	case !strings.HasPrefix(rec.InstanceID, "i-"):
+		return fmt.Errorf("%w: instance_id %q is not an instance ID", ErrInvalidRecord, rec.InstanceID)
+	case rec.InstanceID != key:
+		return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
+	case rec.State != StateLive && rec.State != StateDeleting:
+		return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
+	}
+	if wb := rec.WriteBack; wb != nil {
+		switch {
+		case wb.Node == "":
+			return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
+		case wb.Since.IsZero():
+			return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
+		case !ownsKey(rec.InstanceID, wb.Key):
+			return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
+		case !ownsKey(rec.InstanceID, wb.SeededFrom):
+			return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
+		}
+	}
+	return nil
+}
+
+func bucketConfig() kvstore.Config {
+	return kvstore.Config{
+		Name:        BucketName,
+		Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
+		History:     1,
+		// Left false: a lost bucket has lost obligations, and an empty
+		// recreated one would report every store as having none.
+		RecreateIfMissing: false,
+		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
+			return migrate.DefaultRegistry.RunKV(ctx, BucketName, kv, BucketVersion)
+		},
+		Missing: "UEFI store KV bucket not initialized",
+	}
+}
+
+// Store reads and writes store records. Every change is guarded by the
+// revision the caller read, so a lost race is reported rather than retried.
+type Store struct {
+	kv *kvstore.Store[UEFIStoreRecord]
+}
+
+// NewStore returns a Store over js. The bucket is created on Open or first use.
+func NewStore(js jetstream.JetStream) *Store {
+	return &Store{kv: kvstore.New[UEFIStoreRecord](js, bucketConfig())}
+}
+
+// Open creates the bucket if it is missing and stamps its schema version.
+func (s *Store) Open(ctx context.Context) error {
+	_, err := s.kv.KV(ctx)
+	return err
+}
+
+// Get returns instanceID's record and the revision it was read at. A missing
+// record is kvstore.ErrNotFound; one that breaks its invariants is
+// ErrInvalidRecord, so a newer or corrupt record is never acted on.
+func (s *Store) Get(ctx context.Context, instanceID string) (*UEFIStoreRecord, uint64, error) {
+	rec, rev, err := s.kv.Get(ctx, instanceID)
+	if err != nil {
+		return nil, 0, err
+	}
+	if err := rec.validate(instanceID); err != nil {
+		return nil, 0, err
+	}
+	return rec, rev, nil
+}
+
+// Create writes a new record, which must be live: a store is recorded before
+// its first object is written. An existing record is kvstore.ErrExists. Every
+// write stamps rec with RecordSchemaVersion.
+func (s *Store) Create(ctx context.Context, rec *UEFIStoreRecord) (uint64, error) {
+	rec.SchemaVersion = RecordSchemaVersion
+	if rec.State != StateLive {
+		return 0, fmt.Errorf("%w: %s created in state %q, not %q", ErrInvalidRecord, rec.InstanceID, rec.State, StateLive)
+	}
+	if err := rec.validate(rec.InstanceID); err != nil {
+		return 0, err
+	}
+	return s.kv.Create(ctx, rec.InstanceID, rec)
+}
+
+// CompareAndSet replaces the record only if it is still at rev, returning
+// kvstore.ErrConflict when it is not, and the revision written otherwise.
+func (s *Store) CompareAndSet(ctx context.Context, rec *UEFIStoreRecord, rev uint64) (uint64, error) {
+	rec.SchemaVersion = RecordSchemaVersion
+	if err := rec.validate(rec.InstanceID); err != nil {
+		return 0, err
+	}
+	return s.kv.CompareAndSet(ctx, rec.InstanceID, rec, rev)
+}
+
+// CompareAndDelete removes instanceID's record only if it is still at rev,
+// returning kvstore.ErrConflict when it is not.
+func (s *Store) CompareAndDelete(ctx context.Context, instanceID string, rev uint64) error {
+	return s.kv.CompareAndDelete(ctx, instanceID, rev)
+}

spinifex/domains/ec2/instance/uefistore/uefistore_test.go

@@ -0,0 +1,285 @@
+package uefistore_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const instanceID = "i-0123456789abcdef0"
+
+func openStore(t *testing.T) (*uefistore.Store, jetstream.JetStream) {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	s := uefistore.NewStore(js)
+	require.NoError(t, s.Open(t.Context()))
+	return s, js
+}
+
+func liveRecord() *uefistore.UEFIStoreRecord {
+	return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
+}
+
+func writeBack() *uefistore.UEFIWriteBack {
+	return &uefistore.UEFIWriteBack{
+		Node:       "node-2",
+		Key:        uefistore.KeyPrefix(instanceID) + "b",
+		SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
+		Since:      time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
+	}
+}
+
+func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
+	t.Helper()
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	_, err = kv.Put(t.Context(), key, []byte(body))
+	require.NoError(t, err)
+}
+
+// The records carry obligations past the 1 h terminated-record TTL, so the
+// bucket must not expire them, and it is stamped so a newer schema is refused.
+func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
+	_, js := openStore(t)
+
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	status, err := kv.Status(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, int64(1), status.History())
+	assert.Zero(t, status.TTL(), "a TTL would expire deletion obligations before they are settled")
+
+	v, err := kvutil.ReadVersion(t.Context(), kv)
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.BucketVersion, v)
+}
+
+func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+
+	rev, err := s.Create(t.Context(), rec)
+	require.NoError(t, err)
+
+	got, gotRev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, uefistore.RecordSchemaVersion, got.SchemaVersion)
+	assert.Equal(t, rec, got)
+}
+
+// The wire names are what every release reads, so they are pinned.
+func TestRecord_WireShape(t *testing.T) {
+	s, js := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+	_, err := s.Create(t.Context(), rec)
+	require.NoError(t, err)
+
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	entry, err := kv.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{
+		"schema_version": 1,
+		"instance_id": "i-0123456789abcdef0",
+		"account_id": "111122223333",
+		"state": "live",
+		"write_back": {
+			"node": "node-2",
+			"key": "uefi/i-0123456789abcdef0/b",
+			"seeded_from": "uefi/i-0123456789abcdef0/a",
+			"since": "2026-11-03T01:02:03Z"
+		}
+	}`, string(entry.Value()))
+}
+
+func TestGet_MissingIsNotFound(t *testing.T) {
+	s, _ := openStore(t)
+	_, _, err := s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+func TestCreate_ExistingRecordIsErrExists(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+
+	dup := liveRecord()
+	dup.AccountID = "999999999999"
+	_, err = s.Create(t.Context(), dup)
+	require.ErrorIs(t, err, kvstore.ErrExists)
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, "111122223333", got.AccountID, "the second create must not overwrite the first")
+}
+
+// A store is recorded live before its first object, so a record that starts
+// out deleting has no object it could be describing.
+func TestCreate_RefusesDeleting(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.State = uefistore.StateDeleting
+
+	_, err := s.Create(t.Context(), rec)
+	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
+	s, _ := openStore(t)
+	first, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+
+	moved := liveRecord()
+	moved.WriteBack = writeBack()
+	second, err := s.CompareAndSet(t.Context(), moved, first)
+	require.NoError(t, err)
+	assert.Greater(t, second, first)
+
+	stale := liveRecord()
+	stale.State = uefistore.StateDeleting
+	_, err = s.CompareAndSet(t.Context(), stale, first)
+	require.ErrorIs(t, err, kvstore.ErrConflict)
+
+	got, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, second, rev)
+	assert.Equal(t, uefistore.StateLive, got.State)
+	assert.Equal(t, writeBack(), got.WriteBack)
+}
+
+// Two actors that read the same revision race to change it; exactly one may
+// win, so neither can overwrite an obligation the other recorded.
+func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	_, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+
+	const writers = 8
+	var (
+		wg   sync.WaitGroup
+		mu   sync.Mutex
+		wins []string
+	)
+	for i := range writers {
+		wg.Go(func() {
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
+			rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
+			if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
+				mu.Lock()
+				wins = append(wins, rec.WriteBack.Node)
+				mu.Unlock()
+			} else {
+				assert.ErrorIs(t, err, kvstore.ErrConflict)
+			}
+		})
+	}
+	wg.Wait()
+
+	require.Len(t, wins, 1)
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, wins[0], got.WriteBack.Node)
+}
+
+func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
+	s, _ := openStore(t)
+	first, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	deleting := liveRecord()
+	deleting.State = uefistore.StateDeleting
+	second, err := s.CompareAndSet(t.Context(), deleting, first)
+	require.NoError(t, err)
+
+	require.ErrorIs(t, s.CompareAndDelete(t.Context(), instanceID, first), kvstore.ErrConflict)
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.NoError(t, err, "a delete guarded on a stale revision must leave the record")
+
+	require.NoError(t, s.CompareAndDelete(t.Context(), instanceID, second))
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+// A superseded obligation deletes its recorded key, so a key outside the
+// instance's own prefix must never reach the bucket.
+func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
+	cases := map[string]func(*uefistore.UEFIStoreRecord){
+		"unknown state":            func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
+		"not an instance ID":       func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
+		"write-back with no node":  func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
+		"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
+		"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
+		},
+		"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
+		},
+		"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
+		},
+		"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
+		},
+		"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
+		},
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			s, _ := openStore(t)
+			rev, err := s.Create(t.Context(), liveRecord())
+			require.NoError(t, err)
+
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
+			mutate(rec)
+			_, err = s.CompareAndSet(t.Context(), rec, rev)
+			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+
+			got, gotRev, err := s.Get(t.Context(), instanceID)
+			require.NoError(t, err)
+			assert.Equal(t, rev, gotRev, "an invalid record must not be written")
+			assert.Nil(t, got.WriteBack)
+		})
+	}
+}
+
+// A record a newer release wrote, or one that does not describe its own key,
+// is reported rather than decoded into something a reaper would act on.
+func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
+	cases := map[string]string{
+		"newer schema":       `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"no schema":          `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"another instance":   `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
+		"unknown state":      `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
+		"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
+	}
+	for name, body := range cases {
+		t.Run(name, func(t *testing.T) {
+			s, js := openStore(t)
+			require.True(t, json.Valid([]byte(body)))
+			rawPut(t, js, instanceID, body)
+
+			_, _, err := s.Get(t.Context(), instanceID)
+			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+		})
+	}
+}

spinifex/domains/ec2/launchtemplate/account_cleanup.go

@@ -130,3 +130,51 @@ func tallyOrphan(outcome orphanOutcome, removed, failed, unsettled *int) {
 	case orphanGone, orphanOwned:
 	}
 }
+
+// AccountTemplateDeletes is what one DeleteLiveTemplates run did, by template
+// ID. It is the owner's account of its attempts, not proof of absence.
+type AccountTemplateDeletes struct {
+	DeletedTemplateIDs []string
+	FailedTemplateIDs  []string
+	// Referenced are live templates left because referenced named them, with its reason.
+	Referenced map[string]string
+}
+
+// DeleteLiveTemplates deletes the account's live templates through the same
+// owner delete as DeleteLaunchTemplate, except those referenced names. A header
+// it cannot read or decode is left for the owner scan to report. A listing
+// failure is returned and nothing is deleted.
+func (s *LaunchTemplateServiceImpl) DeleteLiveTemplates(ctx context.Context, accountID string, referenced func(id, name string) string) (*AccountTemplateDeletes, error) {
+	keys, err := kvutil.Keys(ctx, s.kv)
+	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil, fmt.Errorf("list launch-template keys: %w", err)
+	}
+	prefix := accountID + "."
+	out := &AccountTemplateDeletes{Referenced: map[string]string{}}
+	for _, key := range keys {
+		if !isHeaderKey(key, prefix) {
+			continue
+		}
+		entry, err := s.kv.Get(ctx, key)
+		if err != nil {
+			continue
+		}
+		var header LaunchTemplateHeader
+		if json.Unmarshal(entry.Value(), &header) != nil || prefix+header.LaunchTemplateId != key {
+			continue
+		}
+		if reason := referenced(header.LaunchTemplateId, header.LaunchTemplateName); reason != "" {
+			out.Referenced[header.LaunchTemplateId] = reason
+			continue
+		}
+		if err := s.deleteTemplate(ctx, accountID, &header, entry); err != nil {
+			slog.WarnContext(ctx, "Launch-template account cleanup: live template delete failed", "launchTemplateId", header.LaunchTemplateId, "err", err)
+			out.FailedTemplateIDs = append(out.FailedTemplateIDs, header.LaunchTemplateId)
+			continue
+		}
+		out.DeletedTemplateIDs = append(out.DeletedTemplateIDs, header.LaunchTemplateId)
+	}
+	slog.InfoContext(ctx, "Launch-template account live delete completed", "accountID", accountID,
+		"deleted", len(out.DeletedTemplateIDs), "failed", len(out.FailedTemplateIDs), "referenced", len(out.Referenced))
+	return out, nil
+}

spinifex/domains/ec2/snapshot/drain_test.go

@@ -2,12 +2,8 @@ package snapshot
 
 import (
 	"context"
-	"encoding/json"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"net/http"
-	"net/http/httptest"
+	"sync/atomic"
 	"testing"
-	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -15,8 +11,6 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go"
@@ -24,321 +18,78 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const drainInstanceID = "i-drain-host"
+const attachedInstanceID = "i-attached-host"
 
-// setupDrainService returns a snapshot service configured like a daemon that
-// answers ec2.CreateSnapshot: a reachable Predastore host (so drains are not
-// skipped as metadata-only), a data dir with no drain socket in it, and a NATS
-// connection. The Predastore stub rejects everything, so a snapshot that gets
-// past the drain fails fast rather than hanging on a dead address.
-func setupDrainService(t *testing.T) (*SnapshotServiceImpl, *objectstore.MemoryObjectStore, *nats.Conn) {
+// countingSnapshotProvider counts the CreateSnapshot calls it forwards.
+type countingSnapshotProvider struct {
+	ebsprovider.EBSProvider
+
+	creates atomic.Int32
+}
+
+func (p *countingSnapshotProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	p.creates.Add(1)
+	return p.EBSProvider.CreateSnapshot(ctx, req)
+}
+
+// setupAttachedSnapshot is a viperblockd node answering CreateSnapshot for a
+// volume attached to a running instance, with a memory provider behind it.
+func setupAttachedSnapshot(t *testing.T, volumeID string) (*SnapshotServiceImpl, *countingSnapshotProvider, *nats.Conn) {
 	t.Helper()
 
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	t.Cleanup(srv.Close)
-
 	_, nc := testutil.StartTestNATS(t)
-	store := objectstore.NewMemoryObjectStore()
 	cfg := &config.Config{
-		DataDir: testutil.SocketTempDir(t),
-		Predastore: config.PredastoreConfig{
-			Host:   srv.URL,
-			Bucket: "test-bucket",
-		},
+		DataDir:    testutil.SocketTempDir(t),
+		EBS:        config.EBSConfig{Provider: config.EBSProviderViperblockd},
+		Predastore: config.PredastoreConfig{Host: "127.0.0.1:1", Bucket: "test-bucket"},
 	}
-	return NewSnapshotServiceImplWithStore(cfg, store, nc), store, nc
-}
+	svc := NewSnapshotServiceImplWithStore(cfg, objectstore.NewMemoryObjectStore(), nc)
+	provider := &countingSnapshotProvider{EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})}
+	svc.SetEBSProvider(provider)
 
-// seedVolumeAttachment writes the control-plane state.json that decides whether
-// a snapshot must drain.
-func seedVolumeAttachment(t *testing.T, store *objectstore.MemoryObjectStore, volumeID, state, instanceID string) {
-	t.Helper()
-	require.NoError(t, volumestate.Write(context.Background(), store, "test-bucket", volumeID, volumestate.Record{
-		State:            state,
-		AttachedInstance: instanceID,
-	}))
-}
-
-// drainResponder answers the per-instance command subject the way the node
-// hosting the instance does, and hands each request it received to the test.
-func drainResponder(t *testing.T, nc *nats.Conn, instanceID string, reply []byte) chan *nats.Msg {
-	t.Helper()
-
-	got := make(chan *nats.Msg, 4)
-	sub, err := nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
-		got <- msg
-		_ = msg.Respond(reply)
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-
-	return got
-}
-
-// drainedAck is the reply of a node whose drain reached S3.
-func drainedAck(t *testing.T, volumeID string) []byte {
-	t.Helper()
-	data, err := json.Marshal(ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusDrained})
-	require.NoError(t, err)
-	return data
-}
-
-// requireReturnsWithin fails the test if fn has not returned successfully by
-// limit, so a drain that blocks on a socket it should not have dialed reports
-// as a failure rather than as a slow test.
-func requireReturnsWithin(t *testing.T, limit time.Duration, fn func() error) {
-	t.Helper()
-
-	done := make(chan error, 1)
-	go func() { done <- fn() }()
-
-	select {
-	case err := <-done:
-		require.NoError(t, err)
-	case <-time.After(limit):
-		t.Fatalf("drain did not return within %s", limit)
-	}
-}
-
-// awaitDrainCommand returns the command the hosting node received, failing the
-// test if none arrives.
-func awaitDrainCommand(t *testing.T, got chan *nats.Msg) ec2v1.EC2InstanceCommand {
-	t.Helper()
-	select {
-	case msg := <-got:
-		var command ec2v1.EC2InstanceCommand
-		require.NoError(t, json.Unmarshal(msg.Data, &command))
-		assert.Equal(t, testAccountID, msg.Header.Get(natsmsg.AccountIDHeader))
-		return command
-	case <-time.After(2 * time.Second):
-		t.Fatal("the node hosting the volume never received a drain command")
-		return ec2v1.EC2InstanceCommand{}
-	}
-}
-
-// An attached volume is drained on the node hosting its instance, which is the
-// only node with the socket — the node answering CreateSnapshot usually is not.
-func TestDrainVolume_AttachedRoutesToHostingNode(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-attached", "in-use", drainInstanceID)
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-attached"))
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-attached", "", "", testAccountID))
-
-	command := awaitDrainCommand(t, got)
-	assert.True(t, command.Attributes.DrainVolume)
-	assert.Equal(t, drainInstanceID, command.ID)
-	require.NotNil(t, command.DrainVolumeData)
-	assert.Equal(t, "vol-attached", command.DrainVolumeData.VolumeID)
-}
-
-// No responder on the command subject means no node is running the instance,
-// which is what a stopped instance looks like: stop tears down both the plugin
-// and the subscription but deliberately leaves a boot volume attached. Failing
-// here would make every stopped instance's root volume unsnapshottable.
-func TestDrainVolume_AttachedToStoppedInstanceTakesStoppedPath(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-no-host", "in-use", drainInstanceID)
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-no-host", "", "", testAccountID))
-}
-
-// A host that still holds the instance but reports it not running has nothing
-// to flush, so the sealed checkpoint stands. This is the host-drain stop, which
-// unmounts the volumes but keeps the subscription.
-func TestDrainVolume_NotRunningAckTakesStoppedPath(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-not-running", "in-use", drainInstanceID)
-	ack, err := json.Marshal(ec2v1.DrainVolumeResponse{
-		VolumeID: "vol-not-running", Status: ec2v1.DrainVolumeStatusNotRunning,
-	})
-	require.NoError(t, err)
-	got := drainResponder(t, nc, drainInstanceID, ack)
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-not-running", "", "", testAccountID))
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-}
-
-// A hosting node that reports the drain failed (wedged plugin, socket gone
-// mid-restart) fails the snapshot rather than falling through.
-func TestDrainVolume_AttachedWithFailedAckFails(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-drain-err", "in-use", drainInstanceID)
-	drainResponder(t, nc, drainInstanceID,
-		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
-
-	err := svc.drainVolume(context.Background(), "vol-drain-err", "", "", testAccountID)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "drain failed")
-}
-
-// An ack that is not the drained ack is not evidence the writes reached S3.
-func TestDrainVolume_AttachedWithUnexpectedAckFails(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-odd-ack", "in-use", drainInstanceID)
-	drainResponder(t, nc, drainInstanceID, []byte(`{}`))
-
-	err := svc.drainVolume(context.Background(), "vol-odd-ack", "", "", testAccountID)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "unexpected ack")
-}
-
-// Nothing writes to an unattached volume, so the checkpoint its Close() left
-// behind is current and no drain is attempted — not even locally. Dialing the
-// socket is not a probe: it makes the plugin run a full flush, so an available
-// volume must be decided from the record without touching it.
-func TestDrainVolume_AvailableDoesNotDialLocalSocket(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-available", "available", "")
-
-	// A socket that never answers: a dial would park here until the read
-	// deadline, so returning promptly is what proves none was made.
-	testutil.StartBlockingDrainSocket(t, svc.config.DataDir, "vol-available", "OK\n")
-
-	// A nil connection proves no command was issued: routing one would error.
-	svc.natsConn = nil
-
-	requireReturnsWithin(t, 2*time.Second, func() error {
-		return svc.drainVolume(context.Background(), "vol-available", "", "", testAccountID)
-	})
-}
-
-// A volume recorded as in-use but with no instance (drifted state) has no node
-// to address, so it takes the stopped path rather than failing every snapshot.
-func TestDrainVolume_InUseWithoutInstanceTakesStoppedPath(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-no-instance", "in-use", "")
-	svc.natsConn = nil
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-no-instance", "", "", testAccountID))
-}
-
-// When the volume is served by this node the socket is local, so the drain
-// short-circuits without a hop. This is every single-node deployment.
-func TestDrainVolume_LocalSocketShortCircuits(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-local", "in-use", drainInstanceID)
-	testutil.StartDrainSocket(t, svc.config.DataDir, "vol-local", "OK\n")
-
-	// A nil connection proves the local ack was enough: no responder exists for
-	// drainInstanceID, so a routed command would fail.
-	svc.natsConn = nil
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-local", "", "", testAccountID))
-}
-
-// A socket that answers without acking means this node does serve the volume
-// and the flush failed. Routing that onward would come back to this same node
-// and re-run the drain that has already failed, so it fails here instead.
-func TestDrainVolume_LocalSocketErrorAckFailsWithoutRouting(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-local-err", "in-use", drainInstanceID)
-	testutil.StartDrainSocket(t, svc.config.DataDir, "vol-local-err", "ERR\n")
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-local-err"))
-
-	err := svc.drainVolume(context.Background(), "vol-local-err", "", "", testAccountID)
-	require.Error(t, err)
-
-	assert.Empty(t, got, "a local drain that failed must not be re-issued to the host node")
-}
-
-// state.json is the control-plane-owned attachment record; the copy inside
-// config.json is rewritten by the live NBD plugin from its stale in-memory
-// state, so a volume reading "available" there can still be attached.
-func TestDrainVolume_StateJSONOverridesConfigJSON(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-stale-config", "in-use", drainInstanceID)
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-stale-config"))
-
-	// "available" mirrors the stale State a live NBD plugin leaves in
-	// config.json; drainVolume must not trust it (see the test's own doc above).
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-stale-config", "available", "", testAccountID))
-
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-}
-
-// Volumes predating the state.json split have no state object, so the state
-// embedded in config.json is all there is to decide from.
-func TestDrainVolume_FallsBackToConfigJSONWhenNoStateObject(t *testing.T) {
-	svc, _, nc := setupDrainService(t)
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-legacy"))
-
-	// The state/instance embedded in a legacy config.json, with no state.json to overlay it.
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-legacy", "in-use", drainInstanceID, testAccountID))
-
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-}
-
-// Without Predastore the snapshot is metadata-only and never reads a live
-// checkpoint, so there is nothing for a drain to make current.
-func TestDrainVolume_MetadataOnlySkipsDrain(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-meta-only", "in-use", drainInstanceID)
-	svc.config.Predastore.Host = ""
-	svc.natsConn = nil
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-meta-only", "", "", testAccountID))
-}
-
-// putProviderVolume seeds both the control-plane document CreateSnapshot's
-// provider branch reads for attachment state, and the provider's own record,
-// mirroring what CreateVolume's provider branch leaves behind.
-func putProviderVolume(t *testing.T, svc *SnapshotServiceImpl, provider ebsprovider.EBSProvider, volumeID, state, instanceID string) {
-	t.Helper()
-	require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{
-		VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4,
-		State: state, AttachedInstance: instanceID, AvailabilityZone: "us-east-1a",
-		ProviderHandle: "memory://volume/" + volumeID,
-	}))
 	_, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID,
 		CapacityRange:    ebsprovider.CapacityRange{RequiredBytes: 4 * 1024 * 1024 * 1024},
 		AvailabilityZone: "us-east-1a",
 	})
 	require.NoError(t, err)
+	require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{
+		VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4,
+		State: "in-use", AttachedInstance: attachedInstanceID, AvailabilityZone: "us-east-1a",
+	}))
+	return svc, provider, nc
 }
 
-// The provider branch used to skip the drain entirely, silently snapshotting a
-// stale checkpoint of an attached, actively-written volume. It must drain the
-// same way the legacy branch does.
-func TestCreateSnapshot_Provider_DrainsAttachedVolume(t *testing.T) {
-	svc, _, nc := setupDrainService(t)
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
-	svc.SetEBSProvider(provider)
-	putProviderVolume(t, svc, provider, "vol-provider-drain", "in-use", drainInstanceID)
-
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-provider-drain"))
+// The provider drains a buffered volume inside its own CreateSnapshot, so the
+// domain must not dial the drain socket itself: that would drain twice.
+func TestCreateSnapshot_AttachedVolumeOpensNoDrainSocket(t *testing.T) {
+	svc, provider, _ := setupAttachedSnapshot(t, "vol-attached-local")
+	drains := testutil.StartDrainSocket(t, svc.config.DataDir, "vol-attached-local", "OK\n")
 
 	_, err := svc.CreateSnapshot(context.Background(),
-		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider-drain")}, testAccountID)
+		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-attached-local")}, testAccountID)
 	require.NoError(t, err)
 
-	command := awaitDrainCommand(t, got)
-	assert.True(t, command.Attributes.DrainVolume)
-	assert.Equal(t, "vol-provider-drain", command.DrainVolumeData.VolumeID)
+	assert.Zero(t, drains(), "the domain must leave the drain to the provider")
+	assert.Equal(t, int32(1), provider.creates.Load())
 }
 
-// A provider-branch snapshot whose drain fails must fail closed rather than
-// return a snapshot of stale data: the provider must never be asked to
-// snapshot, and no snapshot metadata may be left behind.
-func TestCreateSnapshot_Provider_UndrainableAttachedVolumeFails(t *testing.T) {
-	svc, _, nc := setupDrainService(t)
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
-	svc.SetEBSProvider(provider)
-	putProviderVolume(t, svc, provider, "vol-provider-nodrain", "in-use", drainInstanceID)
-
-	got := drainResponder(t, nc, drainInstanceID,
-		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
-
-	_, err := svc.CreateSnapshot(context.Background(),
-		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider-nodrain")}, testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-
-	out, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{}, testAccountID)
+// With no drain socket on this node, an attached volume is snapshotted by the
+// provider without asking the instance's host for a drain.
+func TestCreateSnapshot_AttachedVolumeSendsItsHostNoDrain(t *testing.T) {
+	svc, provider, nc := setupAttachedSnapshot(t, "vol-attached-remote")
+	var commands atomic.Int32
+	sub, err := nc.Subscribe(ec2v1.InstanceCommandSubject(attachedInstanceID), func(msg *nats.Msg) {
+		commands.Add(1)
+		_ = msg.Respond([]byte(`{"Code":"InternalError","Message":"no drain socket"}`))
+	})
 	require.NoError(t, err)
-	assert.Empty(t, out.Snapshots, "a failed drain must not leave a snapshot behind")
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	snap, err := svc.CreateSnapshot(context.Background(),
+		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-attached-remote")}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, "vol-attached-remote", aws.StringValue(snap.VolumeId))
+	assert.Equal(t, int32(1), provider.creates.Load())
+	assert.Zero(t, commands.Load(), "the instance's host must not be asked to drain")
 }

spinifex/domains/ec2/snapshot/service_impl.go

@@ -5,10 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"log/slog"
-	"net"
-	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -18,10 +15,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
@@ -209,19 +204,12 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
 		return nil, errors.New(awserrors.ErrorIncorrectState)
 	}
 
-	// Flush the writes buffered by whichever node serves the volume, so the
-	// provider reads a current checkpoint. An attached volume that cannot be
-	// drained fails here rather than silently snapshotting stale data.
-	if err := s.drainVolume(ctx, volumeID, volume.State, volume.AttachedInstance, accountID); err != nil {
-		slog.ErrorContext(ctx, "CreateSnapshot: drain failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-
 	created, err := s.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
 		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID, VolumeHandle: volume.ProviderHandle,
 	})
 	if err != nil || created == nil {
-		return nil, errors.New(awserrors.ErrorServerInternal)
+		slog.ErrorContext(ctx, "CreateSnapshot: provider CreateSnapshot failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+		return nil, ProviderCreateError(volumeID, err)
 	}
 	now := time.Now()
 	snapshotCfg := ebsmetadata.Snapshot{
@@ -259,177 +247,6 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
 	return snapshotConfigToEC2(snapshotCfg), nil
 }
 
-const (
-	// drainDialTimeout bounds the connect to the local drain socket. The socket
-	// is either being served on this node or it is not, so a slow connect means
-	// a wedged plugin rather than a busy one.
-	drainDialTimeout = time.Second
-
-	// drainAckTimeout bounds DrainToBackend on the hosting node: it flushes the
-	// WAL and the live checkpoint to S3, so it scales with the dirty set rather
-	// than with a fixed unit of work.
-	drainAckTimeout = 30 * time.Second
-
-	// drainRequestTimeout bounds the ec2.cmd round-trip. It exceeds
-	// drainAckTimeout so a slow drain surfaces as the hosting node's error
-	// instead of an opaque NATS timeout here.
-	drainRequestTimeout = 35 * time.Second
-)
-
-// ErrNoDrainSocket reports that this node has no drain socket for the volume,
-// which is how a node learns it does not serve it. It is deliberately distinct
-// from a socket that answered badly: only the former is worth routing onward,
-// and conflating the two makes a node re-run a drain that has already failed.
-var ErrNoDrainSocket = errors.New("no drain socket on this node")
-
-// DrainVolumeSocket asks the NBD plugin serving volumeID on this node to flush
-// its WAL chunks and live checkpoint to S3, then waits for the ack. The plugin
-// creates the socket under {dataDir}/viperblock/{volumeID}/, so this only
-// succeeds on the node hosting the volume; the daemon calls it when a drain
-// command is routed to it.
-func DrainVolumeSocket(dataDir, volumeID string) error {
-	sockPath := filepath.Join(dataDir, "viperblock", volumeID, "snapshot.sock")
-	conn, err := net.DialTimeout("unix", sockPath, drainDialTimeout)
-	if err != nil {
-		return fmt.Errorf("%w: dial %s: %w", ErrNoDrainSocket, sockPath, err)
-	}
-	defer conn.Close()
-
-	_ = conn.SetDeadline(time.Now().Add(drainAckTimeout))
-	buf := make([]byte, 8)
-	n, err := conn.Read(buf)
-	if err != nil {
-		return fmt.Errorf("read drain ack for %s: %w", volumeID, err)
-	}
-	if !strings.HasPrefix(string(buf[:n]), "OK") {
-		return fmt.Errorf("drain of %s did not ack OK: %q", volumeID, string(buf[:n]))
-	}
-	return nil
-}
-
-// drainVolume flushes the volume's in-flight writes to S3 before the snapshot
-// reads the live checkpoint from there.
-func (s *SnapshotServiceImpl) drainVolume(ctx context.Context, volumeID string, fallbackState, fallbackInstance string, accountID string) error {
-	return DrainVolume(ctx, s.config, s.store, s.natsConn, volumeID, fallbackState, fallbackInstance, accountID)
-}
-
-// DrainVolume flushes the volume's in-flight writes to S3 before a live
-// checkpoint is read from there. Shared by ec2.CreateSnapshot and
-// ec2.CreateImage's running-instance path (domains/ec2/image) so the two
-// live-snapshot call sites cannot diverge on when a drain is required.
-//
-// The drain socket lives on the node serving the volume, but both callers are
-// queue-grouped across every node, so the node answering the request usually
-// is not that node. Whether a drain is required is therefore decided from the
-// volume's attachment record, never from whether the socket happens to be
-// local: an attached volume is being written to right now, and reading its
-// checkpoint without draining silently captures an older one.
-func DrainVolume(ctx context.Context, cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn, volumeID string, fallbackState, fallbackInstance string, accountID string) error {
-	// A metadata-only snapshot never reads the live checkpoint, so there is
-	// nothing for a drain to make current.
-	if cfg == nil || cfg.Predastore.Host == "" {
-		return nil
-	}
-
-	state, instanceID, err := volumeAttachment(ctx, store, cfg.Predastore.Bucket, volumeID, fallbackState, fallbackInstance)
-	if err != nil {
-		return err
-	}
-
-	// Nothing is writing to an unattached volume: the checkpoint its Close()
-	// left behind is the current one. Decide this before touching the socket —
-	// dialing it is not a probe, it makes the plugin run a full flush.
-	if state != "in-use" || instanceID == "" {
-		slog.InfoContext(ctx, "DrainVolume: volume not attached, snapshotting the checkpoint left by Close (stopped instance path)",
-			"volumeId", volumeID, "state", state, "attachedInstance", instanceID)
-		return nil
-	}
-
-	// Fast path: the volume is served by this node, so no hop is needed. This is
-	// every single-node deployment, and the node hosting the instance on a
-	// cluster.
-	localErr := DrainVolumeSocket(cfg.DataDir, volumeID)
-	if localErr == nil {
-		return nil
-	}
-
-	// A socket that answered but did not ack means this node does serve the
-	// volume and the flush itself failed. Routing that onward would land back on
-	// this same node and repeat the drain that has already failed.
-	if !errors.Is(localErr, ErrNoDrainSocket) {
-		return fmt.Errorf("drain volume %s on this node: %w", volumeID, localErr)
-	}
-
-	slog.InfoContext(ctx, "DrainVolume: volume attached, routing drain to the node hosting it",
-		"volumeId", volumeID, "instanceId", instanceID)
-	return drainOnHostNode(ctx, natsConn, volumeID, instanceID, accountID)
-}
-
-// volumeAttachment returns the volume's authoritative state and attached
-// instance. state.json is control-plane-owned and authoritative; the fallback
-// pair (passed in from the caller's own volume record) is rewritten by the
-// live NBD plugin from its stale in-memory state and is only used for volumes
-// predating the split.
-func volumeAttachment(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string, fallbackState, fallbackInstance string) (state, instanceID string, err error) {
-	rec, found, err := volumestate.Read(ctx, store, bucket, volumeID)
-	if err != nil {
-		return "", "", fmt.Errorf("read volume state for %s: %w", volumeID, err)
-	}
-	if found {
-		return rec.State, rec.AttachedInstance, nil
-	}
-	return fallbackState, fallbackInstance, nil
-}
-
-// drainOnHostNode issues the drain on the node hosting instanceID. Only that
-// node subscribes ec2.cmd.{instanceID}, so the command is self-routing and no
-// volume-to-node resolution is needed here.
-//
-// An attachment record is not proof of a live writer: stop deliberately leaves
-// a boot volume attached (runtime/daemon/vm_adapters.go's volumeMounterAdapter.Unmount)
-// while tearing down both the NBD plugin and this subscription. The two
-// not-running signals below are therefore the stopped-instance path, not a
-// failure — treating them as one would make every stopped instance's root
-// volume permanently unsnapshottable.
-func drainOnHostNode(ctx context.Context, natsConn *nats.Conn, volumeID, instanceID, accountID string) error {
-	command := ec2v1.EC2InstanceCommand{
-		ID:              instanceID,
-		Attributes:      ec2v1.EC2CommandAttributes{DrainVolume: true},
-		DrainVolumeData: &ec2v1.DrainVolumeData{VolumeID: volumeID},
-	}
-
-	resp, err := natsmsg.NATSRequest[ec2v1.DrainVolumeResponse](ctx, natsConn,
-		ec2v1.InstanceCommandSubject(instanceID), command, drainRequestTimeout, accountID)
-	if err != nil {
-		// No subscriber at all: the instance runs nowhere in the cluster, so it
-		// was stopped (stop migrates it to shared KV before unsubscribing) and
-		// its volumes were sealed by Close. Warn rather than Info because the
-		// same shape appears if the hosting node has lost NATS while its VM
-		// keeps writing, which yields a stale snapshot as it did before routing
-		// existed. A live host that answers and fails is still a hard error.
-		if errors.Is(err, nats.ErrNoResponders) {
-			slog.WarnContext(ctx, "drainVolume: no node hosts the instance, treating the volume as stopped and snapshotting its sealed checkpoint",
-				"volumeId", volumeID, "instanceId", instanceID)
-			return nil
-		}
-		return fmt.Errorf("drain volume %s on the node hosting %s: %w", volumeID, instanceID, err)
-	}
-
-	switch resp.Status {
-	case ec2v1.DrainVolumeStatusDrained:
-		return nil
-	// The host still holds the instance but it is not running: nothing is
-	// writing, so there is nothing to flush. This is a host-drain stop, which
-	// keeps the VM (and this subscription) in place after unmounting.
-	case ec2v1.DrainVolumeStatusNotRunning:
-		slog.InfoContext(ctx, "drainVolume: instance is not running on its host, snapshotting the volume's sealed checkpoint",
-			"volumeId", volumeID, "instanceId", instanceID)
-		return nil
-	default:
-		return fmt.Errorf("drain volume %s on the node hosting %s: unexpected ack %q", volumeID, instanceID, resp.Status)
-	}
-}
-
 // describeSnapshotsValidFilters defines the set of filter names accepted by DescribeSnapshots.
 var describeSnapshotsValidFilters = map[string]bool{
 	"snapshot-id": true,
@@ -740,6 +557,21 @@ func unreadableRecordError(ctx context.Context, snapshotID, accountID, kind stri
 		"Cannot confirm that snapshot '%s' is not in use: %d %s record(s) could not be read", snapshotID, len(unreadable), kind)
 }
 
+// ProviderCreateError maps a provider CreateSnapshot failure of volumeID to
+// the code a client can act on: retry-later for a provider that cannot answer
+// now, IncorrectState for one that needs the volume detached, else internal.
+func ProviderCreateError(volumeID string, err error) error {
+	switch {
+	case errors.Is(err, ebsprovider.ErrUnavailable):
+		return errors.New(awserrors.ErrorServiceUnavailable)
+	case errors.Is(err, ebsprovider.ErrVolumeInUse):
+		return awserrors.Errorf(awserrors.ErrorIncorrectState,
+			"The volume '%s' is in use. Detach it before creating a snapshot.", volumeID)
+	default:
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+}
+
 // providerDeleteError maps a provider DeleteSnapshot failure to the code a
 // client can act on. A transport failure is retryable: the delete is idempotent.
 func providerDeleteError(snapshotID string, err error) error {

spinifex/domains/ec2/snapshot/service_impl_test.go

@@ -1519,3 +1519,55 @@ func TestDropEmptySnapshotRefsDoesNotDropAConcurrentAdd(t *testing.T) {
 		assert.True(t, has, "iteration %d: the concurrent add was dropped by the empty-key delete", i)
 	}
 }
+
+// createFailingProvider fails only CreateSnapshot, with err.
+type createFailingProvider struct {
+	ebsprovider.EBSProvider
+
+	err error
+}
+
+func (p *createFailingProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, p.err
+}
+
+// A provider that cannot snapshot a volume while it is attached refuses it
+// as in use, which the client can act on by detaching the volume first.
+func TestCreateSnapshot_ProviderVolumeInUseIsIncorrectState(t *testing.T) {
+	svc, store := setupTestSnapshotService(t)
+	createTestVolume(t, svc, store, "vol-create-inuse", 10)
+	svc.SetEBSProvider(&createFailingProvider{EBSProvider: svc.provider,
+		err: &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "published"}})
+
+	_, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-create-inuse")}, testAccountID)
+
+	code, message := clientErrorDetail(t, err)
+	assert.Equal(t, awserrors.ErrorIncorrectState, code)
+	assert.Contains(t, message, "vol-create-inuse")
+	assert.Contains(t, message, "Detach")
+}
+
+// A provider that cannot answer now, such as one refusing a volume another
+// node holds, is retry-later for the client; any other failure is internal.
+func TestCreateSnapshot_ProviderFailureMapsToSpecificError(t *testing.T) {
+	cases := []struct {
+		name     string
+		err      error
+		wantCode string
+	}{
+		{"provider unavailable", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}, awserrors.ErrorServiceUnavailable},
+		{"internal failure", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: "boom"}, awserrors.ErrorServerInternal},
+		{"transport failure", errors.New("connection reset"), awserrors.ErrorServerInternal},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			svc, store := setupTestSnapshotService(t)
+			createTestVolume(t, svc, store, "vol-create-fail", 10)
+			svc.SetEBSProvider(&createFailingProvider{EBSProvider: svc.provider, err: tc.err})
+
+			_, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-create-fail")}, testAccountID)
+			require.Error(t, err)
+			assert.Equal(t, tc.wantCode, err.Error())
+		})
+	}
+}

spinifex/domains/ec2/volume/service_impl_test.go

@@ -1725,6 +1725,28 @@ func TestDeleteVolume_VolumeAttachedButAvailable(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorVolumeInUse, err.Error())
 }
 
+// A stopped instance keeps its volumes in use, so deleting one is refused
+// until it is detached; AWS answers the same.
+func TestDeleteVolume_VolumeHeldByStoppedInstanceIsRefused(t *testing.T) {
+	kv := setupTestVolumeKV(t)
+	store := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
+	svc.snapshotKV = kv
+
+	createVolumeInStoreWithMeta(t, svc, store, "vol-held", ebsmetadata.Volume{
+		VolumeID:         "vol-held",
+		CapacityGiB:      10,
+		State:            "in-use",
+		AttachedInstance: "i-stopped",
+	})
+
+	_, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{
+		VolumeId: aws.String("vol-held"),
+	}, testVolAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorVolumeInUse, err.Error())
+}
+
 func TestDeleteVolume_EmptyStateUnattachedDeletable(t *testing.T) {
 	kv := setupTestVolumeKV(t)
 	store := objectstore.NewMemoryObjectStore()

spinifex/domains/ec2/volumestate/volumestate.go

@@ -1,78 +0,0 @@
-// Package volumestate reads and writes the control-plane-owned attachment
-// state for an EBS volume, persisted to a per-volume state.json object.
-//
-// It is kept out of config.json because config.json is rewritten by the live
-// nbdkit VB on every SaveState (clobbering any State the control plane wrote
-// there) and is a sealed object for encrypted volumes (a second writer reuses
-// the AES-GCM nonce). state.json is plaintext, viperblock never touches it, so
-// the control plane is its single writer. Readers outside the volume service
-// (the snapshot path) need the same record, which is why it lives here.
-package volumestate
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-)
-
-// Record is the control-plane-owned attachment state of a volume.
-type Record struct {
-	State            string    `json:"state"`
-	AttachedInstance string    `json:"attachedInstance"`
-	DeviceName       string    `json:"deviceName"`
-	AttachedAt       time.Time `json:"attachedAt"`
-}
-
-// Key is the S3 key for a volume's control-plane state object.
-func Key(volumeID string) string { return volumeID + "/state.json" }
-
-// Write persists the attachment state, replacing any previous record.
-func Write(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string, rec Record) error {
-	data, err := json.Marshal(rec)
-	if err != nil {
-		return fmt.Errorf("marshal volume state: %w", err)
-	}
-	_, err = store.PutObject(ctx, &s3.PutObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(Key(volumeID)),
-		Body:   bytes.NewReader(data),
-	})
-	if err != nil {
-		return fmt.Errorf("failed to write volume state to S3: %w", err)
-	}
-	return nil
-}
-
-// Read returns the attachment state. found=false with a nil error means the
-// object is absent (a volume predating the state.json split), in which case the
-// caller falls back to the State embedded in config.json.
-func Read(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string) (Record, bool, error) {
-	getResult, err := store.GetObject(ctx, &s3.GetObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(Key(volumeID)),
-	})
-	if err != nil {
-		if objectstore.IsNoSuchKeyError(err) {
-			return Record{}, false, nil
-		}
-		return Record{}, false, fmt.Errorf("failed to get volume state: %w", err)
-	}
-	defer getResult.Body.Close()
-
-	body, err := io.ReadAll(getResult.Body)
-	if err != nil {
-		return Record{}, false, fmt.Errorf("failed to read volume state body: %w", err)
-	}
-	var rec Record
-	if err := json.Unmarshal(body, &rec); err != nil {
-		return Record{}, false, fmt.Errorf("failed to unmarshal volume state: %w", err)
-	}
-	return rec, true, nil
-}

spinifex/domains/ec2/vpc/external_ipam.go

@@ -122,7 +122,7 @@ func (m *ExternalIPAM) AllocateIP(ctx context.Context, region, az, purpose, allo
 	if m == nil {
 		return "", "", fmt.Errorf("external IPAM unavailable on this node: %w", errors.New(awserrors.ErrorInsufficientAddressCapacity))
 	}
-	pool := m.findPool(region, az)
+	pool := external.FindPool(m.pools, region, az)
 	if pool == nil {
 		return "", "", fmt.Errorf("no external pool available for region=%q az=%q: %w", region, az, errors.New(awserrors.ErrorInsufficientAddressCapacity))
 	}
@@ -221,30 +221,6 @@ func (m *ExternalIPAM) allocatorFor(poolName string) (external.Allocator, error)
 	return m.static, nil
 }
 
-// findPool returns the best pool for the given region/AZ using the same
-// fallback order as topology.go: AZ-scoped → region-scoped → unscoped.
-func (m *ExternalIPAM) findPool(region, az string) *external.ExternalPoolConfig {
-	for i := range m.pools {
-		p := &m.pools[i]
-		if p.AZ != "" && p.AZ == az && p.Region == region {
-			return p
-		}
-	}
-	for i := range m.pools {
-		p := &m.pools[i]
-		if p.AZ == "" && p.Region != "" && p.Region == region {
-			return p
-		}
-	}
-	for i := range m.pools {
-		p := &m.pools[i]
-		if p.Region == "" && p.AZ == "" {
-			return p
-		}
-	}
-	return nil
-}
-
 // filterStatic returns only the static pools (Source unset or "static").
 func filterStatic(pools []external.ExternalPoolConfig) []external.ExternalPoolConfig {
 	out := make([]external.ExternalPoolConfig, 0, len(pools))

spinifex/domains/ecr/auth/rotator_test.go

@@ -99,7 +99,7 @@ func TestRotateOnce_MintsAndSwaps(t *testing.T) {
 	r.rotateOnce(t.Context(), time.Now().UTC())
 
 	// A new active key took over.
-	assert.NotEqual(t, key.Kid, issuer.ActiveKid(), "issuer should sign with the rotated key")
+	assert.NotEqual(t, key.Kid, activeKid(issuer), "issuer should sign with the rotated key")
 
 	// Both old and new tokens verify (previous key retained).
 	_, err = verifier.Verify(oldTok)
@@ -142,7 +142,7 @@ func TestRotateOnce_PrunesExpiredKey(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, metas, 1, "the older key should be pruned")
 	assert.Equal(t, second.Kid, active.Kid, "newest key remains active")
-	assert.Equal(t, second.Kid, issuer.ActiveKid())
+	assert.Equal(t, second.Kid, activeKid(issuer))
 }
 
 // TestRotator_RunRotatesThenStops drives the scheduler loop on a fast interval
@@ -169,7 +169,7 @@ func TestRotator_RunRotatesThenStops(t *testing.T) {
 	stopped := make(chan struct{})
 	go func() { r.Run(ctx); close(stopped) }()
 
-	require.Eventually(t, func() bool { return issuer.ActiveKid() != key.Kid },
+	require.Eventually(t, func() bool { return activeKid(issuer) != key.Kid },
 		2*time.Second, 5*time.Millisecond, "rotator should mint a new active key")
 
 	cancel()
@@ -218,3 +218,10 @@ func TestIssuerVerifier_ConcurrentSwap(t *testing.T) {
 	}
 	<-done
 }
+
+// activeKid reads the kid the issuer currently signs with.
+func activeKid(i *Issuer) string {
+	i.mu.RLock()
+	defer i.mu.RUnlock()
+	return i.key.Kid
+}

spinifex/domains/ecr/auth/token.go

@@ -91,16 +91,6 @@ func (i *Issuer) SetActiveKey(key *SigningKey) {
 	i.mu.Unlock()
 }
 
-// ActiveKid returns the kid of the current signing key, or "" if none.
-func (i *Issuer) ActiveKid() string {
-	i.mu.RLock()
-	defer i.mu.RUnlock()
-	if i.key == nil {
-		return ""
-	}
-	return i.key.Kid
-}
-
 // Mint returns a signed ES256 JWT for p and its expiry. The kid header lets any
 // verifier resolve the public key from the shared signing-key set.
 func (i *Issuer) Mint(p Principal) (token string, expiresAt time.Time, err error) {

spinifex/domains/ecr/awsapi/gateway_image_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -25,7 +26,8 @@ import (
 // object store + memory meta) and its composed ECR JSON image-action service.
 func newImageGateway(t *testing.T) *ecrGW {
 	t.Helper()
-	meta := ecrdomain.NewMemoryMetaStore()
+	_, _, js := testutil.StartTestJetStream(t)
+	meta := ecrdomain.NewKVMetaStore(js)
 	reg := ecrregistry.NewRegistry(objectstore.NewMemoryObjectStore(), meta, ecrTestAccount)
 	gw := newECRGW(nil, awsapi.Deps{Registry: awsapi.NewRegistryActionService(reg, reg, reg, reg), LifecyclePreview: awsapi.NewLifecyclePreviewActionService(meta, reg)})
 	imageRegistries.Store(gw, reg)

spinifex/domains/ecr/awsapi/registration.go

@@ -2,14 +2,12 @@ package awsapi
 
 import (
 	"errors"
-	"io"
 	"log/slog"
 	"maps"
 	"net/http"
 	"slices"
 	"strings"
 
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	ecrauth "github.com/mulgadc/spinifex/spinifex/domains/ecr/auth"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -34,7 +32,7 @@ type Deps struct {
 func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
-		ResolveAction: resolveAction,
+		ResolveAction: dispatch.ResolveTargetAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeJSON,
 		Inventory:     OperationInventory(),
@@ -55,25 +53,11 @@ func OperationInventory() dispatch.Inventory {
 	}
 }
 
-// resolveAction is the one X-Amz-Target parser telemetry and dispatch share.
-func resolveAction(r *http.Request) string {
-	return actionFromTarget(r.Header.Get("X-Amz-Target"))
-}
-
-// actionFromTarget extracts the action suffix from an X-Amz-Target header.
-// Any "<Prefix>.<Action>" or bare "<Action>" form is accepted.
-func actionFromTarget(target string) string {
-	if _, after, ok := strings.CutLast(target, "."); ok {
-		return after
-	}
-	return target
-}
-
 // dispatch resolves, scopes and authorizes the request from one read of its
 // body, then runs the action. Errors are awserrors codes the gateway renders.
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
-	action := resolveAction(r)
+	action := dispatch.ResolveTargetAction(r)
 	if action == "" {
 		return errors.New(awserrors.ErrorMissingAction)
 	}
@@ -92,7 +76,7 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 		return errors.New(awserrors.ErrorInternalError)
 	}
 
-	body, err := readBoundedBody(r)
+	body, err := dispatch.ReadBoundedBody(r)
 	if err != nil {
 		slog.Error("ECR dispatch: failed to read body", "err", err)
 		return err
@@ -178,19 +162,6 @@ func roleIDOf(actor dispatch.Actor) string {
 	return roleID
 }
 
-// readBoundedBody reads the body under the SigV4 payload cap, so a body read
-// ahead of the policy gate can neither bypass it nor exhaust memory.
-func readBoundedBody(r *http.Request) ([]byte, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return body, nil
-}
-
 func without(values, excluded []string) []string {
 	return slices.DeleteFunc(slices.Clone(values), func(value string) bool {
 		return slices.Contains(excluded, value)

spinifex/domains/ecr/awsapi/registration_test.go

@@ -172,18 +172,6 @@ var composedCases = []struct{ action, body string }{
 	{"DeleteRepository", `{"repositoryName":"app"}`},
 }
 
-func TestActionFromTarget(t *testing.T) {
-	for target, want := range map[string]string{
-		TargetPrefix + ".CreateRepository": "CreateRepository",
-		"a.b.ListRepositories":             "ListRepositories",
-		"GetAuthorizationToken":            "GetAuthorizationToken",
-		TargetPrefix + ".":                 "",
-		"":                                 "",
-	} {
-		assert.Equal(t, want, actionFromTarget(target), "target %q", target)
-	}
-}
-
 // Every composed action must be in the Actions namespace, or dispatch rejects
 // it as InvalidAction before its resources are ever scoped.
 func TestComposedActionsAreInTheNamespace(t *testing.T) {

spinifex/domains/ecr/meta.go

@@ -5,8 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"maps"
-	"slices"
 	"sort"
 	"strings"
 	"sync"
@@ -593,295 +591,6 @@ func (s *KVMetaStore) DeleteUpload(ctx context.Context, accountID, uploadID stri
 	return nil
 }
 
-// MemoryMetaStore is an in-memory MetaStore used by tests and single-process
-// dev runs. It is safe for concurrent use.
-type MemoryMetaStore struct {
-	mu        sync.Mutex
-	repos     map[string]map[string]RepoMeta     // account -> repo -> meta
-	policies  map[string]map[string][]byte       // account -> repo -> policyText
-	lifecycle map[string]map[string][]byte       // account -> repo -> lifecyclePolicyText
-	tags      map[string]map[string]string       // account -> repo|tag -> digest
-	manifests map[string]map[string]ManifestMeta // account -> repo|digest -> meta
-	uploads   map[string]map[string]uploadRev    // account -> id -> state+rev
-}
-
-type uploadRev struct {
-	state UploadState
-	rev   uint64
-}
-
-var _ MetaStore = (*MemoryMetaStore)(nil)
-
-// NewMemoryMetaStore constructs an empty in-memory MetaStore.
-func NewMemoryMetaStore() *MemoryMetaStore {
-	return &MemoryMetaStore{
-		repos:     make(map[string]map[string]RepoMeta),
-		policies:  make(map[string]map[string][]byte),
-		lifecycle: make(map[string]map[string][]byte),
-		tags:      make(map[string]map[string]string),
-		manifests: make(map[string]map[string]ManifestMeta),
-		uploads:   make(map[string]map[string]uploadRev),
-	}
-}
-
-func (m *MemoryMetaStore) PutRepo(_ context.Context, accountID string, meta RepoMeta) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.repos[accountID] == nil {
-		m.repos[accountID] = make(map[string]RepoMeta)
-	}
-	m.repos[accountID][meta.Name] = meta
-	return nil
-}
-
-func (m *MemoryMetaStore) GetRepo(_ context.Context, accountID, repo string) (RepoMeta, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	r, ok := m.repos[accountID][repo]
-	if !ok {
-		return RepoMeta{}, ErrNotFound
-	}
-	return r, nil
-}
-
-func (m *MemoryMetaStore) ListRepos(_ context.Context, accountID string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return slices.Sorted(maps.Keys(m.repos[accountID])), nil
-}
-
-func (m *MemoryMetaStore) DeleteRepo(_ context.Context, accountID, repo string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if _, ok := m.repos[accountID][repo]; !ok {
-		return ErrNotFound
-	}
-	delete(m.repos[accountID], repo)
-	delete(m.policies[accountID], repo)
-	delete(m.lifecycle[accountID], repo)
-	for k := range m.tags[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo {
-			delete(m.tags[accountID], k)
-		}
-	}
-	for k := range m.manifests[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo {
-			delete(m.manifests[accountID], k)
-		}
-	}
-	return nil
-}
-
-func (m *MemoryMetaStore) ListManifests(_ context.Context, accountID, repo string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	var out []string
-	for k := range m.manifests[accountID] {
-		if r, d, ok := strings.Cut(k, "|"); ok && r == repo {
-			out = append(out, d)
-		}
-	}
-	sort.Strings(out)
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutRepoPolicy(_ context.Context, accountID, repo string, policyText []byte) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.policies[accountID] == nil {
-		m.policies[accountID] = make(map[string][]byte)
-	}
-	m.policies[accountID][repo] = policyText
-	return nil
-}
-
-func (m *MemoryMetaStore) GetRepoPolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.policies[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	return p, nil
-}
-
-func (m *MemoryMetaStore) DeleteRepoPolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.policies[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	delete(m.policies[accountID], repo)
-	return p, nil
-}
-
-func (m *MemoryMetaStore) PutLifecyclePolicy(_ context.Context, accountID, repo string, policyText []byte) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.lifecycle[accountID] == nil {
-		m.lifecycle[accountID] = make(map[string][]byte)
-	}
-	m.lifecycle[accountID][repo] = policyText
-	return nil
-}
-
-func (m *MemoryMetaStore) GetLifecyclePolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.lifecycle[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	return p, nil
-}
-
-func (m *MemoryMetaStore) DeleteLifecyclePolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.lifecycle[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	delete(m.lifecycle[accountID], repo)
-	return p, nil
-}
-
-func (m *MemoryMetaStore) PutTag(_ context.Context, accountID, repo, tag, digest string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.tags[accountID] == nil {
-		m.tags[accountID] = make(map[string]string)
-	}
-	m.tags[accountID][repo+"|"+tag] = digest
-	return nil
-}
-
-func (m *MemoryMetaStore) GetTag(_ context.Context, accountID, repo, tag string) (string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	d, ok := m.tags[accountID][repo+"|"+tag]
-	if !ok {
-		return "", ErrNotFound
-	}
-	return d, nil
-}
-
-func (m *MemoryMetaStore) DeleteTag(_ context.Context, accountID, repo, tag string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	key := repo + "|" + tag
-	if _, ok := m.tags[accountID][key]; !ok {
-		return ErrNotFound
-	}
-	delete(m.tags[accountID], key)
-	return nil
-}
-
-func (m *MemoryMetaStore) ListTags(_ context.Context, accountID, repo string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	var out []string
-	for k := range m.tags[accountID] {
-		if r, t, ok := strings.Cut(k, "|"); ok && r == repo {
-			out = append(out, t)
-		}
-	}
-	sort.Strings(out)
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutManifestMeta(_ context.Context, accountID, repo string, meta ManifestMeta) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.manifests[accountID] == nil {
-		m.manifests[accountID] = make(map[string]ManifestMeta)
-	}
-	m.manifests[accountID][repo+"|"+meta.Digest] = meta
-	return nil
-}
-
-func (m *MemoryMetaStore) GetManifestMeta(_ context.Context, accountID, repo, digest string) (ManifestMeta, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	meta, ok := m.manifests[accountID][repo+"|"+digest]
-	if !ok {
-		return ManifestMeta{}, ErrNotFound
-	}
-	return meta, nil
-}
-
-func (m *MemoryMetaStore) DeleteManifestMeta(_ context.Context, accountID, repo, digest string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	key := repo + "|" + digest
-	if _, ok := m.manifests[accountID][key]; !ok {
-		return ErrNotFound
-	}
-	delete(m.manifests[accountID], key)
-	return nil
-}
-
-func (m *MemoryMetaStore) ListReferrers(_ context.Context, accountID, repo, subjectDigest string) ([]ReferrerDescriptor, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	out := []ReferrerDescriptor{}
-	for k, meta := range m.manifests[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo && meta.SubjectDigest == subjectDigest {
-			out = append(out, meta.referrerDescriptor())
-		}
-	}
-	sort.Slice(out, func(i, j int) bool { return out[i].Digest < out[j].Digest })
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutUpload(_ context.Context, accountID, uploadID string, state UploadState) (uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.uploads[accountID] == nil {
-		m.uploads[accountID] = make(map[string]uploadRev)
-	}
-	rev := m.uploads[accountID][uploadID].rev + 1
-	m.uploads[accountID][uploadID] = uploadRev{state: state, rev: rev}
-	return rev, nil
-}
-
-func (m *MemoryMetaStore) GetUpload(_ context.Context, accountID, uploadID string) (UploadState, uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	u, ok := m.uploads[accountID][uploadID]
-	if !ok {
-		return UploadState{}, 0, ErrNotFound
-	}
-	return u.state, u.rev, nil
-}
-
-func (m *MemoryMetaStore) UpdateUpload(_ context.Context, accountID, uploadID string, state UploadState, rev uint64) (uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	u, ok := m.uploads[accountID][uploadID]
-	if !ok {
-		return 0, ErrNotFound
-	}
-	if u.rev != rev {
-		return 0, ErrConflict
-	}
-	u.state = state
-	u.rev = rev + 1
-	m.uploads[accountID][uploadID] = u
-	return u.rev, nil
-}
-
-func (m *MemoryMetaStore) DeleteUpload(_ context.Context, accountID, uploadID string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if _, ok := m.uploads[accountID][uploadID]; !ok {
-		return ErrNotFound
-	}
-	delete(m.uploads[accountID], uploadID)
-	return nil
-}
-
 // mapKVErr normalizes JetStream KV errors to the MetaStore error vocabulary.
 func mapKVErr(err error) error {
 	if errors.Is(err, jetstream.ErrKeyNotFound) {

spinifex/domains/ecr/meta_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -61,8 +62,8 @@ func TestKVKeyHelpers(t *testing.T) {
 	assert.Equal(t, "sha256-abc", DigestToken("sha256:abc"))
 }
 
-func TestMemoryMetaStore_RepoRoundtrip(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_RepoRoundtrip(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	_, err := m.GetRepo(context.Background(), acct, "team/app")
@@ -80,8 +81,8 @@ func TestMemoryMetaStore_RepoRoundtrip(t *testing.T) {
 	assert.Equal(t, []string{"team/app", "team/web"}, repos)
 }
 
-func TestMemoryMetaStore_TagsAndManifests(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_TagsAndManifests(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	require.NoError(t, m.PutTag(context.Background(), acct, "r", "v1", "sha256:aaa"))
@@ -112,8 +113,8 @@ func TestMemoryMetaStore_TagsAndManifests(t *testing.T) {
 	assert.ErrorIs(t, m.DeleteManifestMeta(context.Background(), acct, "r", "sha256:ccc"), ErrNotFound)
 }
 
-func TestMemoryMetaStore_UploadCAS(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_UploadCAS(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	rev, err := m.PutUpload(context.Background(), acct, "u1", UploadState{RepoName: "r"})
@@ -143,8 +144,8 @@ func TestTrimSuffixMeta(t *testing.T) {
 	assert.Equal(t, "team/app", trimSuffixMeta("repos/team/app/meta"))
 }
 
-func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_DeleteRepoAndListManifests(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	require.NoError(t, m.PutRepo(context.Background(), acct, RepoMeta{Name: "team/app", CreatedAt: time.Now()}))
@@ -156,7 +157,7 @@ func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
 
 	digs, err := m.ListManifests(context.Background(), acct, "team/app")
 	require.NoError(t, err)
-	assert.Equal(t, []string{"sha256:ccc"}, digs)
+	assert.Equal(t, []string{DigestToken("sha256:ccc")}, digs)
 
 	require.NoError(t, m.DeleteRepo(context.Background(), acct, "team/app"))
 	_, err = m.GetRepo(context.Background(), acct, "team/app")
@@ -180,50 +181,6 @@ func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
 	assert.ErrorIs(t, m.DeleteRepo(context.Background(), acct, "team/ghost"), ErrNotFound)
 }
 
-func TestMemoryMetaStore_RepoPolicy(t *testing.T) {
-	m := NewMemoryMetaStore()
-	const acct = "000000000000"
-	const policy = `{"Version":"2012-10-17"}`
-
-	_, err := m.GetRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-	_, err = m.DeleteRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-
-	require.NoError(t, m.PutRepoPolicy(context.Background(), acct, "team/app", []byte(policy)))
-	got, err := m.GetRepoPolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(got))
-
-	deleted, err := m.DeleteRepoPolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(deleted))
-	_, err = m.GetRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-}
-
-func TestMemoryMetaStore_LifecyclePolicy(t *testing.T) {
-	m := NewMemoryMetaStore()
-	const acct = "000000000000"
-	const policy = `{"rules":[]}`
-
-	_, err := m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-	_, err = m.DeleteLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-
-	require.NoError(t, m.PutLifecyclePolicy(context.Background(), acct, "team/app", []byte(policy)))
-	got, err := m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(got))
-
-	deleted, err := m.DeleteLifecyclePolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(deleted))
-	_, err = m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-}
-
 func TestManifestMeta_DecodesRecordWithoutReferrerFields(t *testing.T) {
 	var m ManifestMeta
 	require.NoError(t, json.Unmarshal([]byte(`{"digest":"sha256:ccc","mediaType":"application/vnd.oci.image.manifest.v1+json","size":10,"pushedAt":"2026-01-01T00:00:00Z","childDigests":["sha256:ddd"]}`), &m))
@@ -232,3 +189,9 @@ func TestManifestMeta_DecodesRecordWithoutReferrerFields(t *testing.T) {
 	assert.Empty(t, m.SubjectDigest)
 	assert.Empty(t, m.ArtifactType)
 }
+
+func newTestKVMetaStore(t *testing.T) *KVMetaStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	return NewKVMetaStore(js)
+}

spinifex/domains/ecr/referrers_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -30,11 +29,7 @@ func referrerMeta(digest, subject string) ManifestMeta {
 // its own referrers state.
 func referrerStores(t *testing.T) map[string]MetaStore {
 	t.Helper()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	return map[string]MetaStore{
-		"memory": NewMemoryMetaStore(),
-		"kv":     NewKVMetaStore(testutil.NewJetStream(t, nc)),
-	}
+	return map[string]MetaStore{"kv": newTestKVMetaStore(t)}
 }
 
 func TestListReferrers(t *testing.T) {

spinifex/domains/ecr/registry/lifecycle_sweep_test.go

@@ -50,7 +50,7 @@ func putPolicy(t *testing.T, reg *Registry, account, repo, policy string) {
 const untaggedExpirePolicy = `{"rules":[{"rulePriority":1,"selection":{"tagStatus":"untagged","countType":"sinceImagePushed","countUnit":"days","countNumber":7},"action":{"type":"expire"}}]}`
 
 func TestSweepRepo_SinceImagePushed(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -65,7 +65,7 @@ func TestSweepRepo_SinceImagePushed(t *testing.T) {
 }
 
 func TestSweepRepo_ImageCountMoreThan(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:a", now.AddDate(0, 0, -1), "a")
@@ -81,7 +81,7 @@ func TestSweepRepo_ImageCountMoreThan(t *testing.T) {
 }
 
 func TestSweepRepo_NoPolicy(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:x", now.AddDate(0, 0, -100))
@@ -92,7 +92,7 @@ func TestSweepRepo_NoPolicy(t *testing.T) {
 }
 
 func TestSweepRepo_Idempotent(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -105,7 +105,7 @@ func TestSweepRepo_Idempotent(t *testing.T) {
 }
 
 func TestSweepOnce_MultiAccount(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	const acct2 = "000000000001"
 	policy := `{"rules":[{"rulePriority":1,"selection":{"tagStatus":"any","countType":"imageCountMoreThan","countNumber":1},"action":{"type":"expire"}}]}`
@@ -125,7 +125,7 @@ func TestSweepOnce_MultiAccount(t *testing.T) {
 }
 
 func TestSweepOnce_AccountsError(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	s := NewLifecycleSweeper(reg, func() ([]string, error) {
 		return nil, assert.AnError
 	}, time.Hour)
@@ -133,7 +133,7 @@ func TestSweepOnce_AccountsError(t *testing.T) {
 }
 
 func TestNewLifecycleSweeper_DefaultInterval(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	s := NewLifecycleSweeper(reg, func() ([]string, error) { return nil, nil }, 0)
 	assert.Equal(t, DefaultLifecycleSweepInterval, s.interval)
 
@@ -147,7 +147,7 @@ func TestNewLifecycleSweeper_DefaultInterval(t *testing.T) {
 // TestSweeperRun drives the ticker loop: with a tiny interval and an expiring
 // image, the first tick must delete it; cancelling the context stops Run.
 func TestSweeperRun(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -184,7 +184,7 @@ func TestSweeperRun(t *testing.T) {
 // TestSweepRepo_InvalidStoredPolicy exercises the evaluate-error branch: a
 // malformed stored policy is logged and skipped, deleting nothing.
 func TestSweepRepo_InvalidStoredPolicy(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:x", now.AddDate(0, 0, -100))
@@ -198,7 +198,7 @@ func TestSweepRepo_InvalidStoredPolicy(t *testing.T) {
 // TestSweepRepo_PolicyWithoutRepo exercises the ListImages-not-found branch: a
 // policy stored for a repo with no manifests yields nothing to expire.
 func TestSweepRepo_PolicyWithoutRepo(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	putPolicy(t, reg, testAccount, "ghost", untaggedExpirePolicy)
 

spinifex/domains/ecr/registry/registry_ensurebucket_test.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"testing"
 
-	"github.com/mulgadc/spinifex/spinifex/domains/ecr"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -32,7 +31,7 @@ func (c *countingStore) EnsureBucket(ctx context.Context, bucket string) error {
 
 func TestRegistry_EnsureBucketFailureReturns500(t *testing.T) {
 	store := &countingStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), ensureErr: errors.New("predastore down")}
-	reg := NewRegistry(store, ecr.NewMemoryMetaStore(), testAccount)
+	reg := NewRegistry(store, newTestMetaStore(t), testAccount)
 
 	w := do(reg, http.MethodGet, "/v2/myrepo/tags/list", nil, nil)
 	assert.Equal(t, http.StatusInternalServerError, w.Code)
@@ -40,7 +39,7 @@ func TestRegistry_EnsureBucketFailureReturns500(t *testing.T) {
 
 func TestRegistry_EnsureBucketCachedAfterSuccess(t *testing.T) {
 	store := &countingStore{MemoryObjectStore: objectstore.NewMemoryObjectStore()}
-	reg := NewRegistry(store, ecr.NewMemoryMetaStore(), testAccount)
+	reg := NewRegistry(store, newTestMetaStore(t), testAccount)
 
 	do(reg, http.MethodGet, "/v2/myrepo/tags/list", nil, nil)
 	do(reg, http.MethodGet, "/v2/other/tags/list", nil, nil)

spinifex/domains/ecr/registry/registry_image_test.go

@@ -44,7 +44,7 @@ func storeLayersManifest(t *testing.T, reg *Registry, repo, tag string, layers .
 }
 
 func TestDeleteImage_ReclaimsBlobsKeepsShared(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	shared := pushBlob(t, reg, repo, []byte("shared-layer-bytes"))
 	exclusive := pushBlob(t, reg, repo, []byte("exclusive-layer-bytes"))
@@ -85,7 +85,7 @@ func seedImage(t *testing.T, reg *Registry, repo, tag string) ([]byte, string) {
 }
 
 func TestStoreAndGetManifest_RoundTrip(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	manifest, digest := seedImage(t, reg, "team/app", "v1")
 
 	// Fetch by tag.
@@ -114,7 +114,7 @@ func TestStoreAndGetManifest_RoundTrip(t *testing.T) {
 }
 
 func TestListImages_TaggedAndUntagged(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	seedImage(t, reg, repo, "v1")
 	seedImage(t, reg, repo, "v2")
@@ -176,7 +176,7 @@ func TestListImages_KVDigestForm(t *testing.T) {
 }
 
 func TestDeleteImage_ByTagAndByDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	_, digest := seedImage(t, reg, repo, "v1")
 	seedImage(t, reg, repo, "v2") // shares nothing; distinct digest
@@ -205,7 +205,7 @@ func TestDeleteImage_ByTagAndByDigest(t *testing.T) {
 }
 
 func TestDeleteImage_ByDigestClearsTags(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	_, digest := seedImage(t, reg, repo, "v1")
 	// Add a second tag pointing at the same digest.
@@ -221,7 +221,7 @@ func TestDeleteImage_ByDigestClearsTags(t *testing.T) {
 }
 
 func TestStoreManifest_ImmutableTag(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	manA := fmt.Appendf(nil, `{"schemaVersion":2,"mediaType":"%s","layers":[{"digest":"%s"}]}`,
 		mediaTypeDockerManifest, pushBlob(t, reg, repo, []byte("layer-a")))
@@ -254,7 +254,7 @@ func TestStoreManifest_ImmutableTag(t *testing.T) {
 }
 
 func TestStoreManifest_BadDigestReference(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfgDg := pushBlob(t, reg, repo, []byte("cfg"))
 	manifest := fmt.Appendf(nil,

spinifex/domains/ecr/registry/registry_referrer_test.go

@@ -38,7 +38,7 @@ func storedMeta(t *testing.T, reg *Registry, body string) (subject, artifactType
 }
 
 func TestPutManifest_SubjectRecordedAndAdvertised(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 
 	tests := []struct {
@@ -65,7 +65,7 @@ func TestPutManifest_SubjectRecordedAndAdvertised(t *testing.T) {
 }
 
 func TestPutManifest_IndexSubject(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"manifests":[],"subject":{"mediaType":%q,"digest":%q,"size":10}}`,
@@ -81,7 +81,7 @@ func TestPutManifest_IndexSubject(t *testing.T) {
 }
 
 func TestPutManifest_NoSubjectLeavesFieldsEmpty(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"artifactType":%q,"config":{"mediaType":%q,"digest":%q,"size":3},"layers":[]}`,
 		mediaTypeOCIManifest, sigArtifactType, sigConfigType, cfg)
@@ -97,7 +97,7 @@ func TestPutManifest_NoSubjectLeavesFieldsEmpty(t *testing.T) {
 }
 
 func TestPutManifest_MalformedSubjectDigestRejected(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := imageWithSubject(cfg, "sha256:nothex", "")
 
@@ -111,7 +111,7 @@ func TestPutManifest_MalformedSubjectDigestRejected(t *testing.T) {
 }
 
 func TestPutManifest_IndexWithContentTypeParameterGetsNoConfigArtifactType(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"manifests":[],"subject":{"mediaType":%q,"digest":%q,"size":10}}`,
@@ -128,7 +128,7 @@ func TestPutManifest_IndexWithContentTypeParameterGetsNoConfigArtifactType(t *te
 }
 
 func TestDeleteReferrerManifest_RemovesItFromTheSubjectListing(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := imageWithSubject(cfg, absentSubjectDgt, sigArtifactType)
 	dg := digestOf([]byte(body))
@@ -183,7 +183,7 @@ func getReferrersBody(t *testing.T, reg *Registry, path string) (*httptest.Respo
 }
 
 func TestGetReferrers_ListsReferrersAsAnOCIIndex(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dgA := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	dgB := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, "application/vnd.example.sbom.v1", "cfgB")
 
@@ -202,7 +202,7 @@ func TestGetReferrers_ListsReferrersAsAnOCIIndex(t *testing.T) {
 }
 
 func TestGetReferrers_NoReferrersIsAnEmptyIndexNotAnError(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	w := do(reg, http.MethodGet, "/v2/"+referrerRepo+"/referrers/"+absentSubjectDgt, nil, nil)
@@ -212,7 +212,7 @@ func TestGetReferrers_NoReferrersIsAnEmptyIndexNotAnError(t *testing.T) {
 }
 
 func TestGetReferrers_ArtifactTypeFilter(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dgSig := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	pushReferrer(t, reg, referrerRepo, absentSubjectDgt, "application/vnd.example.sbom.v1", "cfgB")
 
@@ -230,7 +230,7 @@ func TestGetReferrers_ArtifactTypeFilter(t *testing.T) {
 }
 
 func TestGetReferrers_MalformedDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	w := do(reg, http.MethodGet, "/v2/"+referrerRepo+"/referrers/not-a-digest", nil, nil)
@@ -239,14 +239,14 @@ func TestGetReferrers_MalformedDigest(t *testing.T) {
 }
 
 func TestGetReferrers_UnknownRepository(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/ghost/referrers/"+absentSubjectDgt, nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "NAME_UNKNOWN")
 }
 
 func TestGetReferrers_DoesNotCrossRepositories(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	seedRepo(t, reg, "team/other")
 
@@ -256,7 +256,7 @@ func TestGetReferrers_DoesNotCrossRepositories(t *testing.T) {
 }
 
 func TestReferrersRoute_OnlyGET(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 	for _, method := range []string{http.MethodPut, http.MethodDelete, http.MethodPost, http.MethodPatch} {
 		w := do(reg, method, "/v2/"+referrerRepo+"/referrers/"+absentSubjectDgt, []byte("{}"), nil)
@@ -272,7 +272,7 @@ func (failingReferrerStore) ListReferrers(context.Context, string, string, strin
 }
 
 func TestGetReferrers_MetadataFailureIsA5xxNotA404(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 	reg.Meta = failingReferrerStore{reg.Meta}
 
@@ -310,7 +310,7 @@ func referrerDigests(t *testing.T, reg *Registry, repo, subject string) []string
 }
 
 func TestDeleteManifest_SubjectDeleteRemovesItsReferrers(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	sbom := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "sbom")
@@ -326,7 +326,7 @@ func TestDeleteManifest_SubjectDeleteRemovesItsReferrers(t *testing.T) {
 }
 
 func TestDeleteManifest_NestedReferrersAreRemoved(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	sigOfSig := pushReferrer(t, reg, referrerRepo, sig, sigArtifactType, "sig-of-sig")
@@ -341,7 +341,7 @@ func TestDeleteManifest_NestedReferrersAreRemoved(t *testing.T) {
 }
 
 func TestDeleteManifest_ReferrerDeleteLeavesTheSubject(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	other := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "sbom")
@@ -355,7 +355,7 @@ func TestDeleteManifest_ReferrerDeleteLeavesTheSubject(t *testing.T) {
 }
 
 func TestDeleteManifest_TagDeleteDoesNotCascade(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	require.NoError(t, reg.Meta.PutTag(context.Background(), testAccount, referrerRepo, "v1", subject))
@@ -368,7 +368,7 @@ func TestDeleteManifest_TagDeleteDoesNotCascade(t *testing.T) {
 }
 
 func TestDeleteManifest_CascadeStaysInsideTheRepository(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	elsewhere := pushReferrer(t, reg, "team/other", subject, sigArtifactType, "sig-other")
@@ -394,7 +394,7 @@ func (s failingDeleteStore) DeleteManifestMeta(ctx context.Context, account, rep
 }
 
 func TestDeleteManifest_ReferrerFailureDoesNotFailTheSubjectDelete(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	stuck := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "stuck")
 	fine := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "fine")

spinifex/domains/ecr/registry/registry_test.go

@@ -2,6 +2,7 @@ package ecrregistry
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/internal/testkit"
 
 	"crypto/sha256"
 	"encoding/hex"
@@ -21,8 +22,15 @@ import (
 
 const testAccount = "000000000000"
 
-func newTestRegistry() *Registry {
-	return NewRegistry(objectstore.NewMemoryObjectStore(), ecr.NewMemoryMetaStore(), testAccount)
+func newTestRegistry(t *testing.T) *Registry {
+	t.Helper()
+	return NewRegistry(objectstore.NewMemoryObjectStore(), newTestMetaStore(t), testAccount)
+}
+
+func newTestMetaStore(t *testing.T) *ecr.KVMetaStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	return ecr.NewKVMetaStore(js)
 }
 
 func digestOf(b []byte) string {
@@ -69,7 +77,7 @@ func pushBlob(t *testing.T, reg *Registry, repo string, data []byte) string {
 }
 
 func TestBlobUpload_Monolithic_And_Head_Get(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("hello layer bytes")
 	dg := pushBlob(t, reg, "team/app", data)
 
@@ -84,20 +92,20 @@ func TestBlobUpload_Monolithic_And_Head_Get(t *testing.T) {
 }
 
 func TestBlobHead_NotFound(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodHead, "/v2/team/app/blobs/"+digestOf([]byte("absent")), nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }
 
 func TestBlobGet_MalformedDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/blobs/sha256:zzz", nil, nil)
 	assert.Equal(t, http.StatusBadRequest, w.Code)
 	assertCode(t, w, "DIGEST_INVALID")
 }
 
 func TestBlobUpload_Chunked(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	c1 := []byte("first-chunk-")
 	c2 := []byte("second-chunk")
@@ -124,7 +132,7 @@ func TestBlobUpload_Chunked(t *testing.T) {
 }
 
 func TestBlobUpload_DigestMismatch(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	w := do(reg, http.MethodPost, "/v2/r/repo/blobs/uploads/", nil, nil)
 	loc := w.Header().Get("Location")
@@ -134,7 +142,7 @@ func TestBlobUpload_DigestMismatch(t *testing.T) {
 }
 
 func TestBlobUpload_Cancel(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	w := do(reg, http.MethodPost, "/v2/r/repo/blobs/uploads/", nil, nil)
 	loc := w.Header().Get("Location")
@@ -161,7 +169,7 @@ func putManifestReferencing(t *testing.T, reg *Registry, repo, digest string) {
 }
 
 func TestBlobMount_HitAndMiss(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("shared-layer")
 	dg := pushBlob(t, reg, "team/source", data)
 	putManifestReferencing(t, reg, "team/source", dg)
@@ -185,7 +193,7 @@ func TestBlobMount_HitAndMiss(t *testing.T) {
 // as a plain upload start, matching the authorization middleware's
 // classification of the same request.
 func TestBlobMount_MissingFromFallsThroughToUpload(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("shared-layer")
 	dg := pushBlob(t, reg, "team/source", data)
 	putManifestReferencing(t, reg, "team/source", dg)
@@ -200,7 +208,7 @@ func TestBlobMount_MissingFromFallsThroughToUpload(t *testing.T) {
 // named source repository is rejected as a mount — proving digest existence
 // alone is not enough to mount cross-repo.
 func TestBlobMount_UnreferencedDigestFallsThrough(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	unreferenced := pushBlob(t, reg, "team/other", []byte("not-in-source"))
 	seedRepo(t, reg, "team/source")
 	seedRepo(t, reg, "team/dest")
@@ -214,7 +222,7 @@ func TestBlobMount_UnreferencedDigestFallsThrough(t *testing.T) {
 // repository that does not exist never mounts, and never distinguishes
 // itself from an unreferenced-digest miss (both fall through identically).
 func TestBlobMount_NonexistentSourceFallsThrough(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dg := pushBlob(t, reg, "team/elsewhere", []byte("some-bytes"))
 	seedRepo(t, reg, "team/dest")
 
@@ -224,7 +232,7 @@ func TestBlobMount_NonexistentSourceFallsThrough(t *testing.T) {
 }
 
 func TestManifest_PutGetHead_ImageManifest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 
 	cfg := []byte(`{"config":true}`)
@@ -255,7 +263,7 @@ func TestManifest_PutGetHead_ImageManifest(t *testing.T) {
 }
 
 func TestManifest_Put_MissingBlob(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	seedRepo(t, reg, repo)
 	manifest := fmt.Appendf(nil,
@@ -268,7 +276,7 @@ func TestManifest_Put_MissingBlob(t *testing.T) {
 }
 
 func TestManifest_Get_AcceptMismatch(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -285,7 +293,7 @@ func TestManifest_Get_AcceptMismatch(t *testing.T) {
 // acceptable manifest media type as its own Accept header line rather than one
 // comma-joined value. Negotiation must consider every line, not just the first.
 func TestManifest_Get_MultipleAcceptHeaders(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -305,7 +313,7 @@ func TestManifest_Get_MultipleAcceptHeaders(t *testing.T) {
 }
 
 func TestManifest_Get_Unknown(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/manifests/missing", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "MANIFEST_UNKNOWN")
@@ -314,7 +322,7 @@ func TestManifest_Get_Unknown(t *testing.T) {
 // TestStartUpload_RepoNotCreated asserts a blob upload to a repository that was
 // never created is rejected up front — ECR does not auto-create on push.
 func TestStartUpload_RepoNotCreated(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodPost, "/v2/team/ghost/blobs/uploads/", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "NAME_UNKNOWN")
@@ -323,7 +331,7 @@ func TestStartUpload_RepoNotCreated(t *testing.T) {
 // TestPutManifest_RepoNotCreated asserts a manifest PUT to an uncreated
 // repository is rejected with NAME_UNKNOWN.
 func TestPutManifest_RepoNotCreated(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	manifest := fmt.Appendf(nil, `{"mediaType":"%s","layers":[]}`, mediaTypeDockerManifest)
 	w := do(reg, http.MethodPut, "/v2/team/ghost/manifests/v1", manifest,
 		map[string]string{"Content-Type": mediaTypeDockerManifest})
@@ -332,7 +340,7 @@ func TestPutManifest_RepoNotCreated(t *testing.T) {
 }
 
 func TestManifest_ImageIndex_ChildValidation(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 
 	// Push a child image manifest first.
@@ -372,7 +380,7 @@ func TestManifest_ImageIndex_ChildValidation(t *testing.T) {
 }
 
 func TestManifest_Delete_Tag(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -390,7 +398,7 @@ func TestManifest_Delete_Tag(t *testing.T) {
 // TestManifest_Delete_ByDigest proves delete-by-digest is no longer a no-op: it
 // removes the manifest record + object, and a subsequent GET is a 404.
 func TestManifest_Delete_ByDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("cfg-bytes"))
 	manifest := fmt.Appendf(nil,
@@ -412,7 +420,7 @@ func TestManifest_Delete_ByDigest(t *testing.T) {
 }
 
 func TestTagsList(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -434,13 +442,13 @@ func TestTagsList(t *testing.T) {
 }
 
 func TestTagsList_UnknownRepo(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/ghost/tags/list", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }
 
 func TestCatalog(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	pushBlob(t, reg, "team/a", []byte("a"))
 	pushBlob(t, reg, "team/b", []byte("b"))
 
@@ -454,21 +462,21 @@ func TestCatalog(t *testing.T) {
 }
 
 func TestDispatch_InvalidRepoName(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/Team/App/tags/list", nil, nil)
 	assert.Equal(t, http.StatusBadRequest, w.Code)
 	assertCode(t, w, "NAME_INVALID")
 }
 
 func TestDispatch_RepeatedSeparatorRepoName(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dg := pushBlob(t, reg, "team/a__b--c", []byte("layer"))
 	w := do(reg, http.MethodHead, "/v2/team/a__b--c/blobs/"+dg, nil, nil)
 	assert.Equal(t, http.StatusOK, w.Code)
 }
 
 func TestDispatch_UnknownPath(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/frobnicate", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }

spinifex/domains/ecr/registry/registry_upload_test.go

@@ -35,7 +35,7 @@ func (s *blobPutFailStore) PutObject(ctx context.Context, input *s3.PutObjectInp
 func TestFinishUpload_StoreFailure_CleansUp(t *testing.T) {
 	mem := objectstore.NewMemoryObjectStore()
 	store := &blobPutFailStore{MemoryObjectStore: mem}
-	meta := ecr.NewMemoryMetaStore()
+	meta := newTestMetaStore(t)
 	reg := NewRegistry(store, meta, testAccount)
 	seedRepo(t, reg, "team/app")
 
@@ -72,7 +72,7 @@ func awsString(s string) *string { return &s }
 // updates issued from the same observed revision cannot both commit. One wins,
 // the other gets ErrConflict — so a racing PATCH is rejected, never merged.
 func TestUploadCAS_SameRevisionSingleWinner(t *testing.T) {
-	meta := ecr.NewMemoryMetaStore()
+	meta := newTestMetaStore(t)
 	rev, err := meta.PutUpload(context.Background(), testAccount, "u-race", ecr.UploadState{RepoName: "r"})
 	require.NoError(t, err)
 
@@ -116,7 +116,7 @@ func TestUploadCAS_SameRevisionSingleWinner(t *testing.T) {
 // hash state and the recorded byte object never desynchronise.
 func TestPatchUpload_ConcurrentNoCorruption(t *testing.T) {
 	for iter := range 25 {
-		reg := newTestRegistry()
+		reg := newTestRegistry(t)
 		repo := "team/race"
 		seedRepo(t, reg, repo)
 

spinifex/domains/ecr/service_impl.go

@@ -16,8 +16,7 @@ type MetaServiceImpl struct {
 
 var _ MetaService = (*MetaServiceImpl)(nil)
 
-// NewMetaServiceImpl builds a MetaService over an explicit MetaStore (used by
-// daemon-side tests with a MemoryMetaStore).
+// NewMetaServiceImpl builds a MetaService over an explicit MetaStore.
 func NewMetaServiceImpl(store MetaStore) *MetaServiceImpl {
 	return &MetaServiceImpl{store: store}
 }

spinifex/domains/ecr/service_test.go

@@ -21,7 +21,7 @@ const svcTestAccount = "000000000000"
 // ErrNotFound/ErrConflict into the response flags rather than transport errors,
 // so they survive the AWS-error-code-only NATS reply.
 func TestMetaServiceImpl_FlagMapping(t *testing.T) {
-	svc := NewMetaServiceImpl(NewMemoryMetaStore())
+	svc := NewMetaServiceImpl(newTestKVMetaStore(t))
 
 	// Absent repo -> Found:false, nil error.
 	dr, err := svc.RepoDescribe(context.Background(), &RepoDescribeRequest{Repo: "ghost"}, svcTestAccount)

spinifex/domains/ecr/storage.go

@@ -28,42 +28,22 @@ func AccountBucket(accountID string) string {
 // shapes every ECR storage path must use. Blobs are pooled per account for
 // dedup; manifests and tags are indexed per repository.
 const (
-	// repoMetaKeyFmt is the repo config object (scan-on-push, tag mutability).
-	repoMetaKeyFmt = "repos/%s/_meta.json"
-	// tagKeyFmt maps a tag to a digest pointer (body is "sha256:...").
-	tagKeyFmt = "repos/%s/manifests/by-tag/%s"
 	// manifestKeyFmt holds manifest JSON keyed by its own digest.
 	manifestKeyFmt = "repos/%s/manifests/by-digest/%s"
-	// indexKeyFmt is the rolling tag->digest index used by ListImages.
-	indexKeyFmt = "repos/%s/manifests/index.json"
 	// blobKeyFmt is the account-wide blob pool, two-char hex shard.
 	blobKeyFmt = "blobs/%s/%s"
-	// uploadKeyFmt holds in-progress chunked upload bytes.
-	uploadKeyFmt = "uploads/%s"
 	// uploadChunkKeyFmt holds a single revision's assembled bytes under a unique
 	// per-write token, so a CAS-winning PATCH always points at exactly the bytes
 	// it hashed (a losing concurrent write lands on a different, orphaned token).
 	uploadChunkKeyFmt = "uploads/%s/%s"
 )
 
-// RepoMetaKey returns the key for a repository's config object.
-func RepoMetaKey(repo string) string { return fmt.Sprintf(repoMetaKeyFmt, repo) }
-
-// TagKey returns the key for a tag's digest pointer.
-func TagKey(repo, tag string) string { return fmt.Sprintf(tagKeyFmt, repo, tag) }
-
 // ManifestKey returns the key for a manifest stored by its digest. The digest
 // is sanitized via DigestToken so the object key carries no ':' separator.
 func ManifestKey(repo, digest string) string {
 	return fmt.Sprintf(manifestKeyFmt, repo, DigestToken(digest))
 }
 
-// IndexKey returns the key for a repository's tag->digest index.
-func IndexKey(repo string) string { return fmt.Sprintf(indexKeyFmt, repo) }
-
-// UploadKey returns the key for an in-progress upload's bytes.
-func UploadKey(uploadID string) string { return fmt.Sprintf(uploadKeyFmt, uploadID) }
-
 // UploadChunkKey returns the key for one revision's assembled upload bytes,
 // addressed by a unique per-write token.
 func UploadChunkKey(uploadID, token string) string {

spinifex/domains/ecr/storage_test.go

@@ -17,11 +17,7 @@ func TestKeyHelpers(t *testing.T) {
 		got  string
 		want string
 	}{
-		{"repo meta", RepoMetaKey("team/app"), "repos/team/app/_meta.json"},
-		{"tag", TagKey("team/app", "v1"), "repos/team/app/manifests/by-tag/v1"},
 		{"manifest", ManifestKey("team/app", "sha256:abcd"), "repos/team/app/manifests/by-digest/sha256-abcd"},
-		{"index", IndexKey("team/app"), "repos/team/app/manifests/index.json"},
-		{"upload", UploadKey("u-1"), "uploads/u-1"},
 	}
 	for _, c := range cases {
 		if c.got != c.want {

spinifex/domains/ecs/awsapi/registration.go

@@ -2,14 +2,11 @@ package awsapi
 
 import (
 	"errors"
-	"io"
 	"log/slog"
 	"maps"
 	"net/http"
 	"slices"
-	"strings"
 
-	"github.com/mulgadc/bluebottle/pkg/sigv4"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/passrole"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -35,7 +32,7 @@ type Deps struct {
 func NewRegistration(deps Deps) dispatch.Registration {
 	return dispatch.Registration{
 		Service:       ServiceName,
-		ResolveAction: resolveAction,
+		ResolveAction: dispatch.ResolveTargetAction,
 		Dispatch:      deps.dispatch,
 		Errors:        dispatch.ErrorEnvelopeJSON,
 		Inventory:     OperationInventory(),
@@ -50,35 +47,11 @@ func OperationInventory() dispatch.Inventory {
 	}
 }
 
-func resolveAction(r *http.Request) string {
-	return actionFromTarget(r.Header.Get("X-Amz-Target"))
-}
-
-// actionFromTarget extracts the action suffix from an X-Amz-Target header.
-// Any "<Prefix>.<Action>" or bare "<Action>" form is accepted.
-func actionFromTarget(target string) string {
-	if _, after, ok := strings.CutLast(target, "."); ok {
-		return after
-	}
-	return target
-}
-
-func readBoundedBody(r *http.Request) ([]byte, error) {
-	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
-	if err != nil {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	if int64(len(body)) > sigv4.MaxPayloadLen {
-		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
-	}
-	return body, nil
-}
-
 // dispatch resolves, scopes and authorizes the request, then runs the action.
 // Errors are awserrors codes the gateway renders.
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
-	action := resolveAction(r)
+	action := dispatch.ResolveTargetAction(r)
 	if action == "" {
 		return errors.New(awserrors.ErrorMissingAction)
 	}
@@ -99,7 +72,7 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 		return errors.New(awserrors.ErrorInternalError)
 	}
 
-	body, err := readBoundedBody(r)
+	body, err := dispatch.ReadBoundedBody(r)
 	if err != nil {
 		slog.ErrorContext(r.Context(), "ECS dispatch: failed to read body", "err", err)
 		return err

spinifex/domains/ecs/awsapi/registration_target_test.go

@@ -1,13 +0,0 @@
-package awsapi
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestActionFromTarget(t *testing.T) {
-	assert.Equal(t, "RunTask", actionFromTarget("AmazonEC2ContainerServiceV20141113.RunTask"))
-	assert.Equal(t, "CreateCluster", actionFromTarget("CreateCluster"))
-	assert.Empty(t, actionFromTarget(""))
-}

spinifex/domains/ecs/capacity_providers.go

@@ -3,6 +3,7 @@ package ecs
 import (
 	"context"
 	"errors"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -23,7 +24,7 @@ func (s *Service) PutClusterCapacityProviders(ctx context.Context, input *ecs.Pu
 		return nil, err
 	}
 	var rec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -33,7 +34,7 @@ func (s *Service) PutClusterCapacityProviders(ctx context.Context, input *ecs.Pu
 
 	rec.CapacityProviders = awsStringSlice(input.CapacityProviders)
 	rec.DefaultCapacityProviderStrategy = strategyFromAWS(input.DefaultCapacityProviderStrategy)
-	if err := putJSON(ctx, kv, ClusterMetaKey(cluster), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, ClusterMetaKey(cluster), &rec); err != nil {
 		return nil, err
 	}
 	return &ecs.PutClusterCapacityProvidersOutput{Cluster: rec.toAWS()}, nil
@@ -53,7 +54,7 @@ func (s *Service) CreateCapacityProvider(ctx context.Context, input *ecs.CreateC
 	}
 
 	var existing CapacityProviderRecord
-	found, err := getJSON(ctx, kv, CapacityProviderKey(name), &existing)
+	found, err := taskdefinition.GetJSON(ctx, kv, CapacityProviderKey(name), &existing)
 	if err != nil {
 		return nil, err
 	}
@@ -69,7 +70,7 @@ func (s *Service) CreateCapacityProvider(ctx context.Context, input *ecs.CreateC
 		Tags:                     tagsToMap(input.Tags),
 		CreatedAt:                time.Now().UTC(),
 	}
-	if err := putJSON(ctx, kv, CapacityProviderKey(name), &rec); err != nil {
+	if err := taskdefinition.PutJSON(ctx, kv, CapacityProviderKey(name), &rec); err != nil {
 		return nil, err
 	}
 	return &ecs.CreateCapacityProviderOutput{CapacityProvider: rec.toAWS()}, nil
@@ -86,13 +87,13 @@ func (s *Service) DescribeCapacityProviders(ctx context.Context, input *ecs.Desc
 
 	names := awsStringSlice(input.CapacityProviders)
 	if len(names) == 0 {
-		keys, kerr := keysWithPrefix(ctx, kv, CapacityProvidersPrefix())
+		keys, kerr := taskdefinition.KeysWithPrefix(ctx, kv, CapacityProvidersPrefix())
 		if kerr != nil {
 			return nil, kerr
 		}
 		for _, k := range keys {
 			var rec CapacityProviderRecord
-			ok, gerr := getJSON(ctx, kv, k, &rec)
+			ok, gerr := taskdefinition.GetJSON(ctx, kv, k, &rec)
 			if gerr != nil {
 				return nil, gerr
 			}
@@ -106,7 +107,7 @@ func (s *Service) DescribeCapacityProviders(ctx context.Context, input *ecs.Desc
 	for _, ref := range names {
 		name := CapacityProviderShortName(ref)
 		var rec CapacityProviderRecord
-		ok, gerr := getJSON(ctx, kv, CapacityProviderKey(name), &rec)
+		ok, gerr := taskdefinition.GetJSON(ctx, kv, CapacityProviderKey(name), &rec)
 		if gerr != nil {
 			return nil, gerr
 		}
@@ -132,7 +133,7 @@ func (s *Service) DeleteCapacityProvider(ctx context.Context, input *ecs.DeleteC
 		return nil, err
 	}
 	var rec CapacityProviderRecord
-	found, err := getJSON(ctx, kv, CapacityProviderKey(name), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, CapacityProviderKey(name), &rec)
 	if err != nil {
 		return nil, err
 	}

spinifex/domains/ecs/cluster_teardown.go

@@ -3,6 +3,7 @@ package ecs
 import (
 	"context"
 	"errors"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -34,7 +35,7 @@ func (s *Service) DeleteCluster(ctx context.Context, input *ecs.DeleteClusterInp
 		return nil, err
 	}
 	var rec ClusterRecord
-	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, ClusterMetaKey(cluster), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -71,7 +72,7 @@ func (s *Service) DeregisterContainerInstance(ctx context.Context, input *ecs.De
 		return nil, err
 	}
 	var rec InstanceRecord
-	found, err := getJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+	found, err := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 	if err != nil {
 		return nil, err
 	}
@@ -124,7 +125,7 @@ func (s *Service) UpdateContainerInstancesState(ctx context.Context, input *ecs.
 	for _, ref := range awsStringSlice(input.ContainerInstances) {
 		id := ContainerInstanceShortID(ref)
 		var rec InstanceRecord
-		found, gerr := getJSON(ctx, kv, InstanceKey(cluster, id), &rec)
+		found, gerr := taskdefinition.GetJSON(ctx, kv, InstanceKey(cluster, id), &rec)
 		if gerr != nil {
 			return nil, gerr
 		}
@@ -134,7 +135,7 @@ func (s *Service) UpdateContainerInstancesState(ctx context.Context, input *ecs.
 		}
 		rec.Status = status
 		rec.Reaped = false
-		if perr := putJSON(ctx, kv, InstanceKey(cluster, id), &rec); perr != nil {
+		if perr := taskdefinition.PutJSON(ctx, kv, InstanceKey(cluster, id), &rec); perr != nil {
 			return nil, perr
 		}
 		if status == InstanceStatusDraining {

spinifex/domains/ecs/cluster_teardown_test.go

@@ -28,7 +28,7 @@ func TestService_DeleteCluster_CascadesAndSweeps(t *testing.T) {
 	list, err := svc.ListClusters(context.Background(), &ecs.ListClustersInput{}, testAccountID)
 	require.NoError(t, err)
 	assert.Empty(t, list.ClusterArns)
-	keys, err := keysWithPrefix(t.Context(), kv, clusterKeyPrefix("web"))
+	keys, err := taskdefinition.KeysWithPrefix(t.Context(), kv, clusterKeyPrefix("web"))
 	require.NoError(t, err)
 	assert.Empty(t, keys)
 }

spinifex/domains/ecs/deployments_test.go

@@ -19,7 +19,7 @@ import (
 func reloadService(t *testing.T, kv jetstream.KeyValue, cluster, name string) *ServiceRecord {
 	t.Helper()
 	var rec ServiceRecord
-	found, err := getJSON(t.Context(), kv, ServiceKey(cluster, name), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, ServiceKey(cluster, name), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	return &rec
@@ -329,7 +329,7 @@ func TestDeployment_LegacyServiceSynthesizesPrimary(t *testing.T) {
 		DesiredCount: 1, Status: ServiceStatusActive,
 		SchedulingStrategy: SchedulingStrategyReplica, DeploymentID: "legacy-dep",
 	}
-	require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "legacy"), rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ServiceKey("web", "legacy"), rec))
 
 	require.NoError(t, svc.reconcileService(context.Background(), kv, testAccountID, rec))
 	reloaded := reloadService(t, kv, "web", "legacy")
@@ -388,7 +388,7 @@ func TestDeployment_RepeatedStartFailuresBackOffWithoutTheCircuitBreaker(t *test
 	// Once it passes, launching resumes: the backoff is a brake, not a stop.
 	rec = reloadService(t, kv, "web", "web")
 	rec.primaryDeployment().NextLaunchAt = time.Now().UTC().Add(-time.Second)
-	require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "web"), rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ServiceKey("web", "web"), rec))
 	require.NoError(t, svc.reconcileService(context.Background(), kv, testAccountID, reloadService(t, kv, "web", "web")))
 
 	assert.Positive(t, liveServiceTasks(t, svc, kv, "web", "web"), "backoff never released the deployment")

spinifex/domains/ecs/eip_test.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -37,14 +38,14 @@ func TestAssignTaskPublicIP_EnabledAssociatesAndPersists(t *testing.T) {
 	eips := &stubEIPs{publicIP: "192.0.2.50", allocID: "eipalloc-1"}
 	svc.eips = eips
 
-	require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{
 		Name: "web", Cluster: "web", AssignPublicIP: "ENABLED",
 	}))
 	task := &TaskRecord{
 		TaskID: "t-1", Cluster: "web", Group: serviceTaskGroup("web"),
 		ENIID: "eni-1", ENIPrivateIP: "172.31.0.8",
 	}
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), task))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), task))
 
 	svc.assignTaskPublicIP(context.Background(), kv, testAccountID, task)
 
@@ -53,7 +54,7 @@ func TestAssignTaskPublicIP_EnabledAssociatesAndPersists(t *testing.T) {
 	assert.Equal(t, "eipalloc-1", task.ENIEIPAllocationID)
 
 	var persisted TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &persisted)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &persisted)
 	require.NoError(t, err)
 	require.True(t, found)
 	assert.Equal(t, "192.0.2.50", persisted.ENIPublicIP)
@@ -64,7 +65,7 @@ func TestAssignTaskPublicIP_DisabledNoOp(t *testing.T) {
 	eips := &stubEIPs{publicIP: "192.0.2.50", allocID: "eipalloc-1"}
 	svc.eips = eips
 
-	require.NoError(t, putJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, ServiceKey("web", "web"), &ServiceRecord{
 		Name: "web", Cluster: "web", // AssignPublicIP unset
 	}))
 	task := &TaskRecord{

spinifex/domains/ecs/eni_test.go

@@ -189,14 +189,14 @@ func singleTaskRecord(t *testing.T, svc *Service, cluster string) (TaskRecord, b
 	t.Helper()
 	kv, err := svc.bucket(context.Background(), testAccountID)
 	require.NoError(t, err)
-	keys, err := keysWithPrefix(context.Background(), kv, TasksPrefix(cluster))
+	keys, err := taskdefinition.KeysWithPrefix(context.Background(), kv, TasksPrefix(cluster))
 	require.NoError(t, err)
 	require.LessOrEqual(t, len(keys), 1, "test expects at most one task record")
 	if len(keys) == 0 {
 		return TaskRecord{}, false
 	}
 	var rec TaskRecord
-	found, err := getJSON(context.Background(), kv, keys[0], &rec)
+	found, err := taskdefinition.GetJSON(context.Background(), kv, keys[0], &rec)
 	require.NoError(t, err)
 	return rec, found
 }
@@ -317,10 +317,10 @@ func TestReaper_Awsvpc_ReleasesENI(t *testing.T) {
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	var rec InstanceRecord
-	_, err = getJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
+	_, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
 	require.NoError(t, err)
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
 	sc := NewScheduler(nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())

spinifex/domains/ecs/export_test.go

@@ -15,8 +15,6 @@ const TestAccountID = testAccountID
 var (
 	NewTestService   = newTestService
 	RegisterInstance = registerInstance
-	PutJSON          = putJSON
-	GetJSON          = getJSON
 	ServiceTaskGroup = serviceTaskGroup
 )
 

spinifex/domains/ecs/gpu_report.go

@@ -1,6 +1,10 @@
 package ecs
 
-import "context"
+import (
+	"context"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
+)
 
 // ReportTaskGPUInput is the agent's per-task GPU device-assignment report
 // (agent → gateway). Internal agent↔gateway shape, not an AWS SDK action:
@@ -36,7 +40,7 @@ func (s *Service) ReportTaskGPU(ctx context.Context, input *ReportTaskGPUInput,
 		return nil, err
 	}
 	var task TaskRecord
-	found, err := getJSON(ctx, kv, TaskKey(cluster, taskID), &task)
+	found, err := taskdefinition.GetJSON(ctx, kv, TaskKey(cluster, taskID), &task)
 	if err != nil {
 		return nil, err
 	}
@@ -55,7 +59,7 @@ func (s *Service) ReportTaskGPU(ctx context.Context, input *ReportTaskGPUInput,
 		}
 	}
 	if changed {
-		if err := putJSON(ctx, kv, TaskKey(cluster, taskID), &task); err != nil {
+		if err := taskdefinition.PutJSON(ctx, kv, TaskKey(cluster, taskID), &task); err != nil {
 			return nil, err
 		}
 	}

spinifex/domains/ecs/gpu_report_test.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -17,7 +18,7 @@ func TestReportTaskGPU_MergesOntoTaskContainers(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", Cluster: "web",
 		Containers: []ContainerState{
 			{Name: "web", Status: "RUNNING"},
@@ -33,7 +34,7 @@ func TestReportTaskGPU_MergesOntoTaskContainers(t *testing.T) {
 	assert.Equal(t, "OK", out.Acknowledgment)
 
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	for _, c := range rec.Containers {
@@ -63,7 +64,7 @@ func TestDescribeTasks_GPUIDsPopulatedFromReport(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web",
 		Containers: []ContainerState{{Name: "trainer", Status: "RUNNING", GPUIDs: []string{"GPU-aaa", "GPU-bbb"}}},
 	}))
@@ -85,7 +86,7 @@ func TestRecordTaskState_PreservesGPUIDsAcrossStoppedReport(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web",
 		ContainerInstanceID: "i-1",
 		Containers:          []ContainerState{{Name: "trainer", Status: "PENDING"}},
@@ -99,7 +100,7 @@ func TestRecordTaskState_PreservesGPUIDsAcrossStoppedReport(t *testing.T) {
 	}))
 
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.Len(t, rec.Containers, 1)
@@ -114,7 +115,7 @@ func TestRecordTaskState_PreservesGPUIDsAcrossStoppedReport(t *testing.T) {
 		Containers: []ecsv1.ContainerStatus{{Name: "trainer", Status: ecsv1.TaskStatusStopped, ExitCode: &exit}},
 	}))
 
-	found, err = getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err = taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.Len(t, rec.Containers, 1)
@@ -137,7 +138,7 @@ func TestRecordTaskState_ReportedGPUIDsOverwritePinned(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web",
 		ContainerInstanceID: "i-1",
 		Containers:          []ContainerState{{Name: "trainer", Status: "RUNNING", GPUIDs: []string{"GPU-aaa"}}},
@@ -150,7 +151,7 @@ func TestRecordTaskState_ReportedGPUIDsOverwritePinned(t *testing.T) {
 	}))
 
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.Len(t, rec.Containers, 1)
@@ -163,7 +164,7 @@ func TestRecordTaskState_NonGPUContainerStaysEmpty(t *testing.T) {
 	svc, _ := newTestService(t)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	require.NoError(t, putJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, TaskKey("web", "t-1"), &TaskRecord{
 		TaskID: "t-1", ARN: TaskARN(svc.region, testAccountID, "web", "t-1"), Cluster: "web",
 		ContainerInstanceID: "i-1",
 		Containers:          []ContainerState{{Name: "web", Status: "PENDING"}},
@@ -183,7 +184,7 @@ func TestRecordTaskState_NonGPUContainerStaysEmpty(t *testing.T) {
 	}))
 
 	var rec TaskRecord
-	found, err := getJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, TaskKey("web", "t-1"), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	require.Len(t, rec.Containers, 1)

spinifex/domains/ecs/instance_recovery_test.go

@@ -2,6 +2,7 @@ package ecs
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"testing"
 	"time"
 
@@ -27,7 +28,7 @@ func instanceStatus(t *testing.T, svc *Service, cluster, id string) InstanceReco
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	var rec InstanceRecord
-	found, err := getJSON(t.Context(), kv, InstanceKey(cluster, id), &rec)
+	found, err := taskdefinition.GetJSON(t.Context(), kv, InstanceKey(cluster, id), &rec)
 	require.NoError(t, err)
 	require.True(t, found)
 	return rec
@@ -46,7 +47,7 @@ func TestRegister_RestoresReapedInstanceToActive(t *testing.T) {
 	require.NoError(t, err)
 	rec := instanceStatus(t, svc, "web", "i-1")
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
 	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
@@ -86,7 +87,7 @@ func TestReaper_ReleasesCapacityAndRecoversClean(t *testing.T) {
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
 	seeded.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
-	require.NoError(t, putJSON(t.Context(), kv, InstanceKey("web", "i-1"), &seeded))
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &seeded))
 
 	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())