fix: keep the latitude node token readable by the daemon user only

The spinifex group is shared by every service, so group access to the token exposed it. The directory is now root:root 0711 and the file is owned by spinifex-daemon with mode 0400, set on the unchanged path too so an older install converges. Beads mulga-c43h3.3.
fix: close four token and sudo exposures in the latitude deploy
2026-11-03 17:54:42 +00:00 · 2026-11-04 00:51:36 +10:00 · 2026-11-04 00:45:04 +10:00 · 2026-11-04 00:37:04 +10:00 · 2026-11-04 00:17:27 +10:00 · 2026-11-04 00:15:57 +10:00
150 changed files with 11690 additions and 1745 deletions
@@ -0,0 +1,276 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// Dev-only code delivery for the test path: the host attaches the ZIP as a
+// read-only disk and passes its exact size and SHA-256 through fw_cfg. This
+// is not the production delivery, and these bounds are not AWS limits.
+const (
+	entryPackageSize   = "test-package-size"
+	entryPackageSHA256 = "test-package-sha256"
+
+	devMaxZipBytes    = 16 << 20
+	devMaxUnzipBytes  = 64 << 20
+	devMaxEntries     = 1024
+	packageDevice     = "/dev/vda"
+	taskTmpfsSizeOpts = "mode=0755,size=72m"
+)
+
+type packageSpec struct {
+	size   int64
+	sha256 [sha256.Size]byte
+}
+
+type unpackLimits struct {
+	entries int
+	bytes   int64
+}
+
+var devLimits = unpackLimits{entries: devMaxEntries, bytes: devMaxUnzipBytes}
+
+// readPackageSpec reads the expected size and SHA-256 from fw_cfg; it must
+// run before fw_cfg is sealed.
+func readPackageSpec(fwCfgDir string) (packageSpec, error) {
+	var spec packageSpec
+	read := func(name string) (string, error) {
+		b, err := os.ReadFile(filepath.Join(fwCfgDir, name, "raw"))
+		if err != nil {
+			return "", fmt.Errorf("read %s: %w", name, err)
+		}
+		return strings.TrimSpace(string(b)), nil
+	}
+	s, err := read(entryPackageSize)
+	if err != nil {
+		return spec, err
+	}
+	spec.size, err = strconv.ParseInt(s, 10, 64)
+	if err != nil || spec.size <= 0 || spec.size > devMaxZipBytes {
+		return spec, fmt.Errorf("package size %q outside 1..%d bytes", s, devMaxZipBytes)
+	}
+	h, err := read(entryPackageSHA256)
+	if err != nil {
+		return spec, err
+	}
+	b, err := hex.DecodeString(h)
+	if err != nil || len(b) != sha256.Size {
+		return spec, errors.New("package sha256 is not 64 hex digits")
+	}
+	copy(spec.sha256[:], b)
+	return spec, nil
+}
+
+// taskPackage puts the package under dir: a fresh nosuid,nodev tmpfs, made
+// read-only once the files are in place.
+type taskPackage struct {
+	device     string
+	dir        string
+	loadDriver func() error
+	mountDir   func(dir string) error
+	sealDir    func(dir string) error
+}
+
+func devTaskPackage() taskPackage {
+	return taskPackage{device: packageDevice, dir: taskRoot, loadDriver: loadBlockDriver, mountDir: mountTaskDir, sealDir: sealTaskDir}
+}
+
+func (p taskPackage) install(spec packageSpec) error {
+	if err := p.loadDriver(); err != nil {
+		return fmt.Errorf("load block driver: %w", err)
+	}
+	dev, err := os.Open(p.device)
+	if err != nil {
+		return fmt.Errorf("open package disk: %w", err)
+	}
+	defer dev.Close()
+	if err := p.mountDir(p.dir); err != nil {
+		return fmt.Errorf("mount %s: %w", p.dir, err)
+	}
+	if err := unpack(dev, spec, p.dir, devLimits); err != nil {
+		return err
+	}
+	if err := p.sealDir(p.dir); err != nil {
+		return fmt.Errorf("make %s read-only: %w", p.dir, err)
+	}
+	return nil
+}
+
+// unpack reads exactly spec.size bytes once, checks their SHA-256, then
+// validates every entry before writing any of them under dir.
+func unpack(r io.ReaderAt, spec packageSpec, dir string, lim unpackLimits) error {
+	buf := make([]byte, spec.size)
+	if _, err := io.ReadFull(io.NewSectionReader(r, 0, spec.size), buf); err != nil {
+		return fmt.Errorf("read package: %w", err)
+	}
+	if sha256.Sum256(buf) != spec.sha256 {
+		return errors.New("package sha256 mismatch")
+	}
+	zr, err := zip.NewReader(bytes.NewReader(buf), spec.size)
+	if err != nil {
+		return fmt.Errorf("open package: %w", err)
+	}
+	if len(zr.File) > lim.entries {
+		return fmt.Errorf("package has %d entries, limit %d", len(zr.File), lim.entries)
+	}
+	names := make([]string, len(zr.File))
+	var declared int64
+	for i, f := range zr.File {
+		name, err := entryName(f.Name)
+		if err != nil {
+			return err
+		}
+		names[i] = name
+		switch mode := f.Mode(); {
+		case mode.IsDir():
+		case mode.IsRegular():
+			// Compared against what remains, so a huge size cannot wrap the sum.
+			if f.UncompressedSize64 > uint64(lim.bytes-declared) {
+				return fmt.Errorf("%q: unpacked size over the %d-byte limit", f.Name, lim.bytes)
+			}
+			declared += int64(f.UncompressedSize64)
+		default:
+			return fmt.Errorf("%q: %s entries are refused", f.Name, mode.Type())
+		}
+	}
+
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		return err
+	}
+	defer root.Close()
+	var written int64
+	for i, f := range zr.File {
+		n, err := extract(root, names[i], f, lim.bytes-written)
+		written += n
+		if err != nil {
+			return fmt.Errorf("%q: %w", f.Name, err)
+		}
+	}
+	return nil
+}
+
+// extract writes one entry, refusing it if its actual bytes exceed budget
+// whatever its header claims.
+func extract(root *os.Root, name string, f *zip.File, budget int64) (int64, error) {
+	if f.Mode().IsDir() {
+		return 0, mkdirs(root, name)
+	}
+	if d := path.Dir(name); d != "." {
+		if err := mkdirs(root, d); err != nil {
+			return 0, err
+		}
+	}
+	perm := os.FileMode(0o644)
+	if f.Mode()&0o111 != 0 {
+		perm = 0o755
+	}
+	in, err := f.Open()
+	if err != nil {
+		return 0, err
+	}
+	defer in.Close()
+	out, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
+	if err != nil {
+		return 0, err
+	}
+	// The umask applies at create; fchmod does not, so the mode is exact.
+	if err := out.Chmod(perm); err != nil {
+		out.Close()
+		return 0, err
+	}
+	n, err := io.Copy(out, io.LimitReader(in, budget+1))
+	if cerr := out.Close(); err == nil {
+		err = cerr
+	}
+	if err == nil && n > budget {
+		err = errors.New("unpacked bytes over the limit")
+	}
+	return n, err
+}
+
+// mkdirs creates each component of dir as mode 0755 whatever the umask. An
+// existing component must be a real directory, never a symlink.
+func mkdirs(root *os.Root, dir string) error {
+	cur := ""
+	for _, part := range strings.Split(dir, "/") {
+		cur = path.Join(cur, part)
+		if err := root.Mkdir(cur, 0o755); err != nil {
+			if !errors.Is(err, fs.ErrExist) {
+				return err
+			}
+			fi, err := root.Lstat(cur)
+			if err != nil {
+				return err
+			}
+			if !fi.IsDir() {
+				return fmt.Errorf("%q exists and is not a directory", cur)
+			}
+		}
+		if err := root.Chmod(cur, 0o755); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// entryName returns a clean relative path, refusing anything that could name
+// a location outside the task root.
+func entryName(name string) (string, error) {
+	if name == "" || strings.ContainsAny(name, "\\\x00") || path.IsAbs(name) {
+		return "", fmt.Errorf("%q: entry name refused", name)
+	}
+	c := path.Clean(name)
+	if c == "." || c == ".." || strings.HasPrefix(c, "../") {
+		return "", fmt.Errorf("%q: entry name refused", name)
+	}
+	return c, nil
+}
+
+// loadBlockDriver loads virtio_blk, which the image ships but init does not
+// load, then waits briefly for the package disk to appear.
+func loadBlockDriver() error {
+	if err := exec.Command("/sbin/modprobe", "virtio_blk").Run(); err != nil {
+		var uts unix.Utsname
+		if uerr := unix.Uname(&uts); uerr != nil {
+			return uerr
+		}
+		ko := "/lib/modules/" + unix.ByteSliceToString(uts.Release[:]) + "/kernel/drivers/block/virtio_blk.ko"
+		if err := exec.Command("/sbin/insmod", ko).Run(); err != nil {
+			return fmt.Errorf("modprobe and insmod virtio_blk failed: %w", err)
+		}
+	}
+	for range 50 {
+		if _, err := os.Stat(packageDevice); err == nil {
+			return nil
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	return fmt.Errorf("%s did not appear", packageDevice)
+}
+
+func mountTaskDir(dir string) error {
+	return unix.Mount("tmpfs", dir, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, taskTmpfsSizeOpts)
+}
+
+func sealTaskDir(dir string) error {
+	return unix.Mount("", dir, "", unix.MS_REMOUNT|unix.MS_RDONLY|unix.MS_NOSUID|unix.MS_NODEV, "")
+}
@@ -0,0 +1,209 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"hash/crc32"
+	"io/fs"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testZip builds a ZIP; an entry named bootstrap is marked executable.
+func testZip(t *testing.T, files map[string]string) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	names := make([]string, 0, len(files))
+	for n := range files {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	for _, n := range names {
+		h := &zip.FileHeader{Name: n, Method: zip.Deflate}
+		h.SetMode(0o644)
+		if n == "bootstrap" {
+			h.SetMode(0o755)
+		}
+		w, err := zw.CreateHeader(h)
+		require.NoError(t, err)
+		_, err = w.Write([]byte(files[n]))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+// rawZip writes entries with headers set by the caller, so a test can claim
+// sizes or modes the data does not have.
+func rawZip(t *testing.T, entries ...*zip.FileHeader) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for _, h := range entries {
+		w, err := zw.CreateRaw(h)
+		require.NoError(t, err)
+		_, err = w.Write([]byte("x"))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+func specFor(b []byte) packageSpec {
+	return packageSpec{size: int64(len(b)), sha256: sha256.Sum256(b)}
+}
+
+// Not parallel: the umask is process-wide, so the test sets a restrictive one
+// and restores it.
+func TestUnpackInstallsFilesWithFixedModes(t *testing.T) {
+	old := syscall.Umask(0o077)
+	t.Cleanup(func() { syscall.Umask(old) })
+	b := testZip(t, map[string]string{"bootstrap": "#!/bin/sh\n", "lib/data.txt": "data"})
+	dir := t.TempDir()
+	require.NoError(t, unpack(bytes.NewReader(b), specFor(b), dir, devLimits))
+
+	pfi, err := os.Stat(filepath.Join(dir, "lib"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o755), pfi.Mode().Perm(), "implicit parent dir")
+
+	fi, err := os.Stat(filepath.Join(dir, "bootstrap"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o755), fi.Mode().Perm(), "executable")
+	fi, err = os.Stat(filepath.Join(dir, "lib", "data.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o644), fi.Mode().Perm())
+	got, err := os.ReadFile(filepath.Join(dir, "lib", "data.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, "data", string(got))
+}
+
+func TestUnpackRefusesBeforeWritingAnything(t *testing.T) {
+	symlink := &zip.FileHeader{Name: "link", Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: 1}
+	symlink.SetMode(fs.ModeSymlink | 0o777)
+	fifo := &zip.FileHeader{Name: "pipe", Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: 1}
+	fifo.SetMode(fs.ModeNamedPipe | 0o644)
+	// The second size would wrap a plain running sum back under the limit.
+	big := func(name string, n uint64) *zip.FileHeader {
+		return &zip.FileHeader{Name: name, Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: n}
+	}
+	cases := map[string]struct {
+		zip  []byte
+		want string
+	}{
+		"parent traversal": {testZip(t, map[string]string{"a": "ok", "../escape": "x"}), "entry name refused"},
+		"nested traversal": {testZip(t, map[string]string{"a": "ok", "lib/../../escape": "x"}), "entry name refused"},
+		"absolute path":    {testZip(t, map[string]string{"a": "ok", "/etc/escape": "x"}), "entry name refused"},
+		"backslash":        {testZip(t, map[string]string{"a": "ok", `..\escape`: "x"}), "entry name refused"},
+		"symlink":          {rawZip(t, big("a", 1), symlink), "entries are refused"},
+		"named pipe":       {rawZip(t, big("a", 1), fifo), "entries are refused"},
+		"declared size":    {rawZip(t, big("a", devMaxUnzipBytes+1)), "over the"},
+		"wrapping sizes":   {rawZip(t, big("a", devMaxUnzipBytes), big("b", math.MaxUint64)), "over the"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := t.TempDir()
+			err := unpack(bytes.NewReader(tc.zip), specFor(tc.zip), dir, devLimits)
+			assert.ErrorContains(t, err, tc.want)
+			left, rerr := os.ReadDir(dir)
+			require.NoError(t, rerr)
+			assert.Empty(t, left, "files written before the refusal")
+		})
+	}
+}
+
+func TestUnpackRefusesTooManyEntries(t *testing.T) {
+	b := testZip(t, map[string]string{"a": "1", "b": "2", "c": "3"})
+	err := unpack(bytes.NewReader(b), specFor(b), t.TempDir(), unpackLimits{entries: 2, bytes: devMaxUnzipBytes})
+	assert.ErrorContains(t, err, "limit 2")
+}
+
+// The header claims a size within the limit, but the data is longer: the
+// actual bytes are refused, whatever the header says.
+func TestUnpackRefusesBytesBeyondTheLimit(t *testing.T) {
+	data := bytes.Repeat([]byte("a"), 64)
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	h := &zip.FileHeader{Name: "a", Method: zip.Store, CRC32: crc32.ChecksumIEEE(data), CompressedSize64: uint64(len(data)), UncompressedSize64: 8}
+	w, err := zw.CreateRaw(h)
+	require.NoError(t, err)
+	_, err = w.Write(data)
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	b := buf.Bytes()
+
+	dir := t.TempDir()
+	err = unpack(bytes.NewReader(b), specFor(b), dir, unpackLimits{entries: 4, bytes: 16})
+	require.Error(t, err)
+	fi, serr := os.Stat(filepath.Join(dir, "a"))
+	if serr == nil {
+		assert.LessOrEqual(t, fi.Size(), int64(17))
+	}
+}
+
+func TestUnpackChecksSizeAndHash(t *testing.T) {
+	b := testZip(t, map[string]string{"bootstrap": "x"})
+	wrongHash := specFor(b)
+	wrongHash.sha256[0] ^= 1
+	assert.ErrorContains(t, unpack(bytes.NewReader(b), wrongHash, t.TempDir(), devLimits), "sha256 mismatch")
+
+	longer := specFor(b)
+	longer.size++
+	assert.ErrorContains(t, unpack(bytes.NewReader(b), longer, t.TempDir(), devLimits), "read package")
+}
+
+func TestReadPackageSpecBounds(t *testing.T) {
+	sum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	cases := map[string]struct {
+		size, hash string
+		want       string
+	}{
+		"valid":     {"10", sum, ""},
+		"zero":      {"0", sum, "outside"},
+		"too large": {"16777217", sum, "outside"},
+		"not hex":   {"10", "zz", "64 hex digits"},
+		"short":     {"10", sum[:62], "64 hex digits"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := t.TempDir()
+			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash} {
+				require.NoError(t, os.MkdirAll(filepath.Join(dir, n), 0o700))
+				require.NoError(t, os.WriteFile(filepath.Join(dir, n, "raw"), []byte(v), 0o600))
+			}
+			spec, err := readPackageSpec(dir)
+			if tc.want == "" {
+				require.NoError(t, err)
+				assert.Equal(t, int64(10), spec.size)
+				return
+			}
+			assert.ErrorContains(t, err, tc.want)
+		})
+	}
+}
+
+// A pre-existing symlink where a directory belongs is refused, not followed
+// or chmodded.
+func TestMkdirsRefusesSymlinkComponent(t *testing.T) {
+	dir := t.TempDir()
+	outside := t.TempDir()
+	require.NoError(t, os.Chmod(outside, 0o700))
+	require.NoError(t, os.Symlink(outside, filepath.Join(dir, "lib")))
+	root, err := os.OpenRoot(dir)
+	require.NoError(t, err)
+	defer root.Close()
+	assert.ErrorContains(t, mkdirs(root, "lib/sub"), "not a directory")
+	fi, err := os.Stat(outside)
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o700), fi.Mode().Perm(), "symlink target was chmodded")
+}
@@ -30,6 +30,7 @@ var handler = flag.String("handler", os.Getenv("_HANDLER"), "function handler pa
 // connection; the carrier is not the guest protocol.
 type testlink struct {
 	startup
+	pkg            taskPackage
 	runtimeAddr    string
 	startBootstrap func(ctx context.Context, env []string) (*exec.Cmd, error)
 }
@@ -37,6 +38,7 @@ type testlink struct {
 func start(ctx context.Context) int {
 	t := testlink{
 		startup:     defaultStartup(),
+		pkg:         devTaskPackage(),
 		runtimeAddr: runtimeAddr,
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			return agent.StartBootstrap(ctx, taskRoot+"/bootstrap", env, os.Stdout, os.Stderr)
@@ -50,11 +52,16 @@ func start(ctx context.Context) int {
 }

 func (t testlink) run(parent context.Context) (int, error) {
+	// The package values are fw_cfg entries, so read them before connect seals it.
+	spec, specErr := readPackageSpec(t.fwCfgDir)
 	conn, code, err := t.connect(parent)
 	if err != nil {
-		return code, err
+		return code, errors.Join(err, specErr)
 	}
 	defer conn.Close()
+	if specErr != nil {
+		return exitRefused, specErr
+	}
 	link := hostlink.NewTestStream(conn)

 	// The handshake can return before the host refuses the guest, so nothing
@@ -69,6 +76,9 @@ func (t testlink) run(parent context.Context) (int, error) {
 	if parent.Err() != nil {
 		return exitLink, context.Cause(parent)
 	}
+	if err := t.pkg.install(spec); err != nil {
+		return exitRefused, fmt.Errorf("install package: %w", err)
+	}

 	srv := runtimeapi.New()
 	ln, err := net.Listen("tcp", t.runtimeAddr)
@@ -5,10 +5,14 @@ package main
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"io"
 	"net/http"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"strconv"
 	"testing"
 	"time"

@@ -43,12 +47,31 @@ func TestHelperBootstrap(t *testing.T) {
 type bootstrapStarts struct {
 	n       int
 	started chan struct{}
+	task    string
+	sealed  bool
 }

-func (b *bootstrapStarts) testlink(g *guest) testlink {
+// testlink builds a guest whose fw_cfg also carries a controlled package,
+// installed into a temp dir without mounting anything.
+func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) testlink {
+	t.Helper()
+	zipBytes := testZip(t, map[string]string{"bootstrap": "#!/bin/sh\n", "lib/data.txt": "data"})
+	disk := filepath.Join(t.TempDir(), "disk")
+	require.NoError(t, os.WriteFile(disk, zipBytes, 0o600))
+	sum := sha256.Sum256(zipBytes)
+	entries[entryPackageSize] = []byte(strconv.Itoa(len(zipBytes)))
+	entries[entryPackageSHA256] = []byte(hex.EncodeToString(sum[:]))
+	g := newGuest(t, entries)
 	b.started = make(chan struct{}, 1)
+	b.task = t.TempDir()
 	return testlink{
-		startup:     g.startup(),
+		startup: g.startup(),
+		pkg: taskPackage{
+			device: disk, dir: b.task,
+			loadDriver: func() error { return nil },
+			mountDir:   func(string) error { return nil },
+			sealDir:    func(string) error { b.sealed = true; return nil },
+		},
 		runtimeAddr: "127.0.0.1:0",
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			b.n++
@@ -80,7 +103,7 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))

 	conn := h.guest(t)
 	defer conn.Close()
@@ -109,6 +132,8 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		assert.Error(t, err, "agent answered a second invocation")
 	}
 	assert.Equal(t, 1, b.n)
+	assert.True(t, b.sealed, "task dir not made read-only")
+	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
 }

 func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
@@ -117,7 +142,7 @@ func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))

 	conn := h.guest(t)
 	defer conn.Close()
@@ -142,7 +167,7 @@ func TestTestlinkHostRefusalNeverStartsBootstrap(t *testing.T) {
 	entries[lambdav1.EntryGuestKey], entries[lambdav1.EntryGuestCert] = wrong[lambdav1.EntryGuestKey], wrong[lambdav1.EntryGuestCert]
 	var b bootstrapStarts

-	res := wait(t, runTestlinkAsync(ctx, b.testlink(newGuest(t, entries))), "agent to stop")
+	res := wait(t, runTestlinkAsync(ctx, b.testlink(t, entries)), "agent to stop")
 	assert.Equal(t, exitLink, res.code)
 	require.Error(t, res.err)
 	assert.Contains(t, res.err.Error(), "bad certificate")
@@ -155,7 +180,7 @@ func TestTestlinkHostCloseBeforeFrameNeverStartsBootstrap(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))

 	require.NoError(t, h.guest(t).Close())
 	res := wait(t, done, "agent to stop")
@@ -177,3 +202,45 @@ func TestOneInvocationHonoursCancelledContext(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "r-1", req.RequestID)
 }
+
+// A package that fails its check is refused after authentication, and the
+// bootstrap never starts.
+func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	tl := b.testlink(t, h.entries())
+	tl.pkg.device = filepath.Join(t.TempDir(), "other")
+	require.NoError(t, os.WriteFile(tl.pkg.device, testZip(t, map[string]string{"bootstrap": "x"}), 0o600))
+	done := runTestlinkAsync(ctx, tl)
+
+	conn := h.guest(t)
+	defer conn.Close()
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{RequestID: "r-1"}))
+	res := wait(t, done, "agent to stop")
+	assert.Equal(t, exitRefused, res.code)
+	assert.ErrorContains(t, res.err, "install package")
+	assert.Zero(t, b.n)
+	assert.False(t, b.sealed)
+}
+
+// Missing package values still seal fw_cfg, and the agent refuses before
+// waiting for any frame.
+func TestTestlinkMissingPackageSpecRefuses(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	tl := b.testlink(t, h.entries())
+	require.NoError(t, os.RemoveAll(filepath.Join(tl.fwCfgDir, entryPackageSHA256)))
+
+	res := wait(t, runTestlinkAsync(ctx, tl), "agent to stop")
+	assert.Equal(t, exitRefused, res.code)
+	assert.ErrorContains(t, res.err, "read "+entryPackageSHA256)
+	assert.Zero(t, b.n)
+	_, err := os.Stat(tl.fwCfgRoot)
+	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg not sealed")
+}
@@ -25,6 +25,10 @@ const (
 	DeletionStateRunning   = "RUNNING"
 	DeletionStateCompleted = "COMPLETED"
 	DeletionStateFailed    = "FAILED"
+
+	// DeletionStateExercised ends a non-finalizing run: the account is left
+	// TERMINATING and a later DeleteAccount takes the same operation over.
+	DeletionStateExercised = "EXERCISED"
 )

 // CreateAccountRequest is the /admin/CreateAccount request body.
@@ -52,6 +56,10 @@ type DeleteAccountRequest struct {
 	ClientToken string `json:"clientToken"`
 	Force       bool   `json:"force,omitempty"`
 	DryRun      bool   `json:"dryRun,omitempty"`
+
+	// NonFinalizing quiesces the account, cleans the quiesced EC2 platform
+	// kinds and stops. No legacy stage runs and the account record stays.
+	NonFinalizing bool `json:"nonFinalizing,omitempty"`
 }

 // DescribeAccountDeletionRequest is the /admin/DescribeAccountDeletion body.
@@ -76,8 +84,8 @@ type ListAccountsResponse struct {
 }

 // IAMQuiescence is the coordinator's recorded view of IAM quiescence for one
-// fence. Recording only: it admits nothing, and QuiescedScope stays empty until
-// a final inventory re-read exists to catch a late write.
+// fence. Recording only: it admits nothing. QuiescedScope is filled only for a
+// non-finalizing run, and holds as of EvaluatedAt: a write after that is not caught.
 type IAMQuiescence struct {
 	OperationID   string            `json:"operationId"`
 	Fence         uint64            `json:"fence"`
@@ -136,3 +144,67 @@ const (
 	ReasonOwnerInventoryIncomplete     = "owner-inventory-incomplete"
 	ReasonFinalInventoryRereadNotWired = "final-inventory-reread-not-wired"
 )
+
+// ExerciseResult is what a non-finalizing run did after IAM quiescence.
+type ExerciseResult struct {
+	// Skipped is the fixed reason no cleanup ran; Cleanup is then absent.
+	Skipped    string           `json:"skipped,omitempty"`
+	Cleanup    *PlatformCleanup `json:"cleanup,omitempty"`
+	Accounting []KindAccounting `json:"accounting,omitempty"`
+}
+
+// PlatformCleanup is the EC2 platform cleanup result. It carries resource IDs,
+// fixed reasons and counts only, never names, tags, key material or data.
+type PlatformCleanup struct {
+	Kinds  []CleanupKind  `json:"kinds"`
+	Items  []CleanupItem  `json:"items,omitempty"`
+	Counts []CleanupCount `json:"counts,omitempty"`
+}
+
+// CleanupKind is one kind's scan status after the cleanup's own re-scan.
+type CleanupKind struct {
+	Kind              string   `json:"kind"`
+	Status            string   `json:"status"`
+	IncompleteReasons []string `json:"incompleteReasons,omitempty"`
+}
+
+// CleanupItem is one resource by its ID with what the cleanup did to it.
+type CleanupItem struct {
+	ResourceID  string `json:"resourceId"`
+	Kind        string `json:"kind"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	MemberCount int    `json:"memberCount,omitempty"`
+}
+
+// CleanupCount is the number of records of one class that have no public ID.
+type CleanupCount struct {
+	Kind        string `json:"kind"`
+	Class       string `json:"class"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	Count       int    `json:"count"`
+}
+
+// KindAccounting is the final inventory re-read held against the cleanup result
+// for one kind. Quiescence is not rewritten from it.
+type KindAccounting struct {
+	Kind      string `json:"kind"`
+	Status    string `json:"status"`
+	Reason    string `json:"reason,omitempty"`
+	Remaining int    `json:"remaining"`
+}
+
+// Accounting statuses and fixed reasons of a KindAccounting.
+const (
+	AccountingAccounted   = "accounted"
+	AccountingUnaccounted = "unaccounted"
+
+	AccountingReasonRereadIncomplete = "reread-incomplete"
+	AccountingReasonUnnamedRemainder = "unnamed-remainder"
+)
+
+// Fixed reasons an exercise ran no cleanup.
+const (
+	ExerciseSkippedNoQuiescedKind = "no-quiesced-kind"
+)
@@ -43,6 +43,27 @@ func TestAdminWire_DeletionStates(t *testing.T) {
 	require.Equal(t, "RUNNING", DeletionStateRunning)
 	require.Equal(t, "COMPLETED", DeletionStateCompleted)
 	require.Equal(t, "FAILED", DeletionStateFailed)
+	require.Equal(t, "EXERCISED", DeletionStateExercised)
+}
+
+func TestAdminWire_NonFinalizingDeleteRequest(t *testing.T) {
+	goldenJSON(t, DeleteAccountRequest{AccountID: "000000000042", AccountName: "acme", ClientToken: "tok-3", NonFinalizing: true},
+		`{"accountId":"000000000042","accountName":"acme","clientToken":"tok-3","nonFinalizing":true}`)
+}
+
+func TestAdminWire_ExerciseResult(t *testing.T) {
+	goldenJSON(t, ExerciseResult{
+		Cleanup: &PlatformCleanup{
+			Kinds:  []CleanupKind{{Kind: "key-pair", Status: "observed-clean-at-scan-time"}},
+			Items:  []CleanupItem{{ResourceID: "pg-1", Kind: "placement-group", Disposition: "blocked", Reason: "instance-members", MemberCount: 1}},
+			Counts: []CleanupCount{{Kind: "key-pair", Class: "public-key-object-without-metadata", Disposition: "deleted", Count: 2}},
+		},
+		Accounting: []KindAccounting{{Kind: "placement-group", Status: AccountingAccounted, Remaining: 1}},
+	}, `{"cleanup":{"kinds":[{"kind":"key-pair","status":"observed-clean-at-scan-time"}],
+		"items":[{"resourceId":"pg-1","kind":"placement-group","disposition":"blocked","reason":"instance-members","memberCount":1}],
+		"counts":[{"kind":"key-pair","class":"public-key-object-without-metadata","disposition":"deleted","count":2}]},
+		"accounting":[{"kind":"placement-group","status":"accounted","remaining":1}]}`)
+	goldenJSON(t, ExerciseResult{Skipped: ExerciseSkippedNoQuiescedKind}, `{"skipped":"no-quiesced-kind"}`)
 }

 func TestAdminWire_ListAccountsResponse(t *testing.T) {
@@ -65,9 +65,10 @@ The EC2 registration (`spinifex/domains/ec2/awsapi`, `NewRegistration`) parses t
 The gateway communicates with daemons via NATS request/response. Most calls go through `natsmsg.NATSRequest`, which marshals the input, attaches the account ID as a NATS header, and unmarshals the typed response:

 ```go
-func (s *NATSInstanceService) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
-    topic := fmt.Sprintf("ec2.RunInstances.%s", aws.StringValue(input.InstanceType))
-    return natsmsg.NATSRequest[ec2.Reservation](ctx, s.natsConn, topic, input, 5*time.Minute, accountID)
+func runIntoReservation(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, accountID, crID string) (*ec2.Reservation, error) {
+    subject := "ec2.RunInstances.cr." + crID
+    reservation, err := natsmsg.NATSRequest[ec2.Reservation](ctx, natsConn, subject, input, 5*time.Minute, accountID)
+    // ...
 }
 ```

@@ -59,6 +59,9 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM

 - **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
 - **The cluster security group is auto-managed.** As on EKS, Spinifex creates one `eks-cluster-sg-<name>-<id>` in your VPC, tagged `kubernetes.io/cluster/<name>=owned` and `aws:eks:cluster-name`, allowing all traffic from itself. It is attached to the workers and the private-endpoint network interface, and `DescribeCluster` reports it as `clusterSecurityGroupId`. A launch template that names security groups replaces it on the workers, as on AWS. Clusters created before this change keep their `-control-plane-sg` and `-nodegroup-sg` until you recreate them. Deleting a cluster deletes the cluster security group and removes only the rules on your other security groups that reference it. `vpc_config.security_group_ids` is ignored.
+- **Node group deletes are asynchronous.** `DeleteNodegroup` returns at once with the group `DELETING`, and `DescribeNodegroup` answers `ResourceNotFoundException` once the workers and the group's instance profile are gone. Deleting a cluster or creating a group of the same name while a group is `DELETING` returns `ResourceInUseException`, as does a second delete of a `DELETING` group.
+- **If a node group delete cannot finish,** the group moves to `DELETE_FAILED` after 30 minutes, with a `NODE_TERMINATION_FAILURE` health issue naming the workers that are still running. Run `DeleteNodegroup` again to resume it.
+- **Each node group owns an EKS-managed instance profile.** Spinifex creates it, tags it `spinifex:eks-cluster`, `spinifex:eks-nodegroup` and `spinifex:eks-profile-id`, and puts the node role in it. It is removed with the group, so you can delete the node role afterwards with no extra steps. Spinifex never deletes an instance profile it did not create, or one whose tags, name or role no longer match what it recorded; it leaves the profile and logs a warning with its name.
 - **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
 - **Workers launch with IMDSv2 required and an IMDS hop limit of 2.**
  This matches AWS managed node groups and lets a pod without IRSA reach the node role through the SDK default credential chain.
@@ -377,6 +380,15 @@ Control plane readiness and kubectl authentication above lists which clusters re

 **`create-cluster` rejected with an authentication-mode error.** `CONFIG_MAP` is the one mode Spinifex rejects, because it has no `aws-auth` ConfigMap path. Set `authentication_mode` to `API` or `API_AND_CONFIG_MAP`, or leave it unset, and grant access with access entries.

+**`delete-role` fails with `DeleteConflict` after deleting a node group.** Node groups created before per-group instance profiles left an instance profile named after the role, which Spinifex no longer removes because it cannot prove it created it. Remove it yourself:
+
+```bash
+aws iam remove-role-from-instance-profile --instance-profile-name <role-name> --role-name <role-name>
+aws iam delete-instance-profile --instance-profile-name <role-name>
+```
+
+**A node group is `DELETE_FAILED`.** `aws eks describe-nodegroup` lists the workers that did not stop under `health.issues`. Once they are gone, or after you fix the cause, run `aws eks delete-nodegroup` again.
+
 **Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, use the AWS form from `DescribeAddonVersions` (see Add-on versions above), or leave `addon_version` unset so Spinifex selects the catalog default.

 ## Reference Architectures
@@ -34,7 +34,7 @@ Node group workers and CSI-provisioned EBS volumes are never removed by a cluste

 ## Gaps

-1. Four `eks.*` handlers still hold their request for work that grows with the resource, and the subject table names them as not conformant: `CreateCluster` (reclaiming a `FAILED` cluster of the same name tears its infra down inline), `DeleteNodegroup` (one awaited terminate per worker), `UpdateNodegroupConfig` (one awaited launch or terminate per worker added or removed) and `RestoreSnapshot` (launches a control-plane VM and fences the old one inline).
+1. Three `eks.*` handlers still hold their request for work that grows with the resource, and the subject table names them as not conformant: `CreateCluster` (reclaiming a `FAILED` cluster of the same name tears its infra down inline), `UpdateNodegroupConfig` (one awaited launch or terminate per worker added or removed) and `RestoreSnapshot` (launches a control-plane VM and fences the old one inline).
 2. AWS's answer to a repeat delete on a `DELETING` cluster has not been observed; `DELETING` from the record is the chosen behaviour.
 3. Control-plane, load balancer and VPC resources in the system account are keyed by the bare cluster name, so a teardown by name can reach another tenant's same-named cluster.
 4. The teardown deletes load balancers the in-cluster controller created in the customer account, which AWS leaves behind.
@@ -265,7 +265,12 @@ If this returns a list of available instance types, your installation is working

 Spinifex is running, but it holds nothing yet: no machine images, no networks, no instances.

-Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance.
+Log in to node 1, and follow [Setting Up Your Cluster](/docs/setting-up-your-cluster) there to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance:
+
+```bash
+./deploy-vultr.sh ssh
+```
+
 An instance you launch into a public subnet gets a Vultr reserved IP, which you can see in the Vultr portal under **Network → Reserved IPs**.

 ## 8. Test resiliency
@@ -260,6 +260,14 @@ out() {
    ssh "${SSH_OPTS[@]}" "$SSH_USER@$host" "$@" 2>/dev/null || true
 }

+# remote_join_cmd prints the command that joins one node. The token is not in it: it arrives on
+# stdin and is read after sudo has logged its command, so sudo's log never holds it. The other
+# values are positional arguments of a literal script, never spliced into it.
+remote_join_cmd() {
+    local script='exec spx admin join --force --node "$1" --bind "$2" --cluster-bind "$2" --advertise "$3" --host "$4" --token "$(cat)" --region "$5" --az "$6"'
+    printf 'sudo bash -c %q _ %q %q %q %q %q %q' "$script" "$@"
+}
+
 SETUP_OVN="/usr/local/share/spinifex/setup-ovn.sh"

 # --- Preflight -------------------------------------------------------------
@@ -652,14 +660,18 @@ fi
 [ -n "$EXT_IFACE" ] && init_args+=(--external-iface="$EXT_IFACE")
 [ -n "$EMAIL" ] && init_args+=(--email="$EMAIL")

-INIT_LOG=/tmp/spx-init.log
+# init prints the join token, so its log is root-only and outside the world-writable /tmp.
+INIT_LOG=/var/log/spx-init.log
+
+# redact_tokens hides join tokens in text about to be shown to the operator.
+redact_tokens() { sed -E 's/spx_join_[A-Za-z0-9_-]+/spx_join_[redacted]/g'; }

 log "initializing ${HOSTS[0]} as ${NODE_NAMES[0]} (waiting for $((N - 1)) node(s) to join)"

 # init blocks in a formation server until every node has joined, so it runs
 # detached and the token is read back out of its output.
 on "${HOSTS[0]}" "sudo rm -f $INIT_LOG; \
-    sudo setsid sh -c 'spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
+    sudo setsid sh -c 'umask 077; spx admin init ${init_args[*]} >$INIT_LOG 2>&1' </dev/null >/dev/null 2>&1 & \
    exit 0" || fail "${HOSTS[0]}: could not start spx admin init"

 TOKEN=""
@@ -675,7 +687,7 @@ else
        elapsed=$((elapsed + 5))
    done
    if [ -z "$TOKEN" ]; then
-        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | redact_tokens | sed 's/^/  | /' >&2
        fail "${HOSTS[0]}: spx admin init produced no join token within ${TOKEN_TIMEOUT}s"
    fi
    log "  join token issued"
@@ -691,11 +703,9 @@ fi
 # waiting on a shell that node 2 is still holding.
 JOIN_PIDS=()
 for i in $(seq 1 $((N - 1))); do
-    on "${HOSTS[$i]}" "sudo spx admin join --force \
-        --node ${NODE_NAMES[$i]} \
-        --bind ${LAN_IPS[$i]} --cluster-bind ${LAN_IPS[$i]} --advertise ${WAN_IPS[$i]} \
-        --host ${LAN_IPS[0]}:$PORT --token $TOKEN \
-        --region $REGION --az $AZ" &
+    # The token goes to the node on stdin, so it is in no ssh argument and not in sudo's log.
+    # spx admin join takes it only as --token, so it is still in that process's arguments.
+    on "${HOSTS[$i]}" "$(remote_join_cmd "${NODE_NAMES[$i]}" "${LAN_IPS[$i]}" "${WAN_IPS[$i]}" "${LAN_IPS[0]}:$PORT" "$REGION" "$AZ")" <<<"$TOKEN" &
    JOIN_PIDS+=("$!")
 done

@@ -724,7 +734,7 @@ if ! $DRY_RUN; then
        elapsed=$((elapsed + 5))
    done
    if [ "$elapsed" -ge "$FORMATION_TIMEOUT" ]; then
-        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | sed 's/^/  | /' >&2
+        out "${HOSTS[0]}" "sudo tail -40 $INIT_LOG" | redact_tokens | sed 's/^/  | /' >&2
        fail "${HOSTS[0]}: formation did not complete within ${FORMATION_TIMEOUT}s"
    fi
    log "  formation complete"
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Tests remote_join_cmd from install-node.sh: the join token must reach spx only on stdin, so
+# sudo's log (which holds the command it was given) never contains it, and the other values must
+# arrive as plain arguments even when they hold shell syntax.
+# preflight runs this with sh
+[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
+set -euo pipefail
+here="$(cd "$(dirname "$0")" && pwd)"
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+mkdir "$work/bin"
+cd "$work"
+
+cat > "$work/bin/sudo" <<'SH'
+#!/bin/sh
+echo "$*" >> "$FAKE_SUDO_LOG"
+exec "$@"
+SH
+cat > "$work/bin/spx" <<'SH'
+#!/bin/sh
+for a in "$@"; do printf '<%s>' "$a"; done > "$FAKE_SPX_ARGV"
+SH
+chmod +x "$work/bin/sudo" "$work/bin/spx"
+export FAKE_SUDO_LOG="$work/sudo.log" FAKE_SPX_ARGV="$work/spx.argv" PATH="$work/bin:$PATH"
+
+eval "$(sed -n '/^remote_join_cmd()/,/^}/p' "$here/install-node.sh")"
+
+fails=0
+check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
+
+TOKEN=spx_join_FAKETOKEN-abc_1
+cmd="$(remote_join_cmd node2 10.0.0.2 203.0.113.2 '10.0.0.1:4432;touch PWNED' us-east-1 'az $(touch PWNED2)')"
+bash -c "$cmd" <<<"$TOKEN"
+
+check "the token is not in the command" "$(printf '%s' "$cmd" | grep -c FAKETOKEN || true)" 0
+check "the token is not in sudo's log" "$(grep -c FAKETOKEN "$FAKE_SUDO_LOG" || true)" 0
+check "spx got the token" "$(grep -o '<--token><[^>]*>' "$FAKE_SPX_ARGV")" "<--token><$TOKEN>"
+check "spx got the other values as given" "$(cat "$FAKE_SPX_ARGV")" "<admin><join><--force><--node><node2><--bind><10.0.0.2><--cluster-bind><10.0.0.2><--advertise><203.0.113.2><--host><10.0.0.1:4432;touch PWNED><--token><$TOKEN><--region><us-east-1><--az><az \$(touch PWNED2)>"
+check "no value was run as a command" "$(find "$work" -maxdepth 1 -name 'PWNED*' | wc -l | tr -d ' ')" 0
+
+[ "$fails" -eq 0 ]
@@ -0,0 +1,10 @@
+.terraform/
+terraform.tfstate
+terraform.tfstate.*
+*.tfplan
+*.auto.tfvars
+crash.log
+terraform.tfvars
+*.pem
+*.log
+.terraform.lock.hcl
@@ -0,0 +1,132 @@
+# latitude-spx
+
+Builds a Spinifex deployment on latitude.sh bare metal from your laptop with `deploy-latitude.sh`.
+The commands, flags, step markers and state layout follow `vultr-spx`.
+
+**Status: not yet run end to end.**
+The script was written from the plan doc findings (`docs/development/feature/latitude-bm.md`, L1 to L26) and from the provider schema.
+No latitude.sh API call or host run was made while building it, so treat the first `up` as the test.
+One node has been the only shape run live, and only in part (see Unverified). The three-node flow, including the floating pool, has not been run: Ben takes that over.
+
+## What it creates
+
+| Resource | Notes |
+|---|---|
+| Project | `spinifex-<name>`, unless `--project` names an existing one (then it is never deleted) |
+| SSH key | `spinifex-<name>`, account scoped |
+| User data | `spinifex-<name>`, host preparation only (no `{{ }}`, latitude.sh substitutes them) |
+| Servers | `m4-metal-medium`, site `DAL`, Ubuntu 26.04, hourly billing, hostnames `spx-<name>-nodeN` |
+| Elastic IP | one by default (`elastic_ip_count` in `variables.tf`, 1 to 5; set `TF_VAR_elastic_ip_count` to change it). With one node it is a static pool address; with three nodes the addresses form the `latitude` floating pool, which Spinifex moves between servers |
+| Virtual network | three nodes only: `spinifex-<name>`, one VLAN assignment per server |
+
+## Requirements
+
+OpenTofu (or Terraform) 1.5 or later, `aws` CLI, `python3`, `curl`, `ssh`, an SSH key pair.
+
+| Variable | When | Meaning |
+|---|---|---|
+| `LATITUDESH_AUTH_TOKEN` | always | the account API token; used by the provider and this script only, never sent to a node |
+| `LATITUDESH_NODE_TOKEN` | `--nodes 3` | a second, different, writable API key, copied to each node (`/etc/spinifex/latitude/api-token`, owned by `spinifex-daemon` with mode 0400, in a root-owned 0711 directory; the other services share the `spinifex` group, so no group may read it; ownership and mode are set on every `up`; a symlink or any other owner there is refused) so the daemon can move Elastic IPs |
+
+The script refuses a node key it can see is read-only (it lists `/auth/api_keys` and matches the key's last five characters). If the key is not among the account's keys, write authority is first proven by the first Elastic IP move.
+
+Exposure: a latitude.sh API key is team-wide. It is not scoped to Elastic IPs, so anyone who reads it from a node can do whatever the key's owner can in the team (the API exposes `read_only` and `allowed_ips` on a key, and no narrower scope). Create the node key with `allowed_ips` set to the nodes' public addresses, and revoke it after `destroy`.
+
+Nodes are reached as `ubuntu` with the deployment's SSH key, and remote commands run through passwordless `sudo`: latitude.sh puts the key on the image's login user and root login is not enabled.
+
+## Usage
+
+```
+export LATITUDESH_AUTH_TOKEN=...        # from the latitude.sh dashboard
+./deploy-latitude.sh up --name demo     # one node
+
+export LATITUDESH_NODE_TOKEN=...        # a second, writable key
+./deploy-latitude.sh up --name demo3 --nodes 3
+./deploy-latitude.sh status --name demo
+./deploy-latitude.sh ssh --name demo node1 'systemctl status spinifex.target'
+./deploy-latitude.sh credentials --name demo
+./deploy-latitude.sh stop node1 --name demo
+./deploy-latitude.sh start node1 --name demo
+./deploy-latitude.sh destroy --name demo
+```
+
+Options: `--name`, `--site` (alias `--region`), `--plan`, `--os`, `--project ID|SLUG`, `--distro TARBALL`, `--channel`, `--spx-region`, `--spx-az`, `--ssh-key`, `--image`, `--data-disk-id`, `--vpc-subnet` (the VLAN's /24, three nodes), `--nodes 1|3`, `--yes`.
+`./deploy-latitude.sh --help` lists them with defaults.
+
+## What `up` does
+
+`terraform` then, per node: `host` (wait for SSH as `ubuntu` and cloud-init, `/dev/kvm`), `disk`, `vlan` (three nodes), `install`, `form`, `start`; then `pool` (one node) or `cluster` (three nodes: `install-node.sh` forms the cluster); three nodes then install the node token on every `up` and add the `latitude` pool (`pool`); then `restart` (three nodes wait for the current daemon to log "floating pool ready" for the pool), `public`, `image` and the AWS profile.
+Each step writes `step.<name>[.nodeN]` in the state directory, and a rerun resumes at the first step without one.
+
+- **Data disk.** `host-data-disk.sh` picks the blank NVMe by content and by-id EUI, never by `nvmeXn1` (L1, L11).
+It needs exactly one candidate of at least 100 GB with no signature, partitions, holders or mounts, or it stops and lists the disks (`--data-disk-id` chooses).
+It mounts by UUID with `nofail` and the script records the UUID in `data_uuid.<node>`.
+- **Mount check.** Before anything writes under `/var/lib/spinifex`, and on every run, the mounted filesystem UUID must equal the recorded one.
+The units that write there also get a `RequiresMountsFor=/var/lib/spinifex` drop-in.
+The mount is by UUID in fstab, so renamed disks do not change it, and there is no check at service start.
+- **Firewall.** `setup.sh --firewall on` runs before Spinifex first starts.
+- **One node.** The Elastic IP is a static pool (`gateway_ip = "192.0.2.1"`, the node's next hop as `gateway`, no `prefix_len`) with the IMDS addresses moved off the ones latitude.sh routes.
+- **Three nodes.** The `vlan` step makes `eno2.<vid>` with `ip link` and persists it as systemd-networkd files for that link plus a drop-in on the `eno2` unit.
+It never runs `netplan apply`, refuses a subnet that overlaps the SSH peer or a route, and removes the VLAN again if the public interface's address, default route or the route to the SSH peer differ afterwards.
+`form_cluster` then puts the cluster and Geneve planes on that VLAN with `install-node.sh` (`--user ubuntu`); its output is not kept, as it can hold the join token.
+- **Floating pool (three nodes).** The `latitude` pool names the Elastic IP IDs and the project ID from the Terraform outputs, and each node's `provider_host_id` is its server ID. `spinifex.toml` is edited idempotently and read back. Failover moves an address with one `PATCH /elastic_ips/{id}` and waits for it to be active; latitude.sh has no unassign, so a node that loses an address is not told to release it.
+- **State** is in `~/.spinifex/latitude/<name>/`: Terraform state, `ids` (every `latitudesh_*` object in the state, merged on each apply and before each destroy), `step.*` markers, logs and the data disk UUIDs.
+- **AWS profile** `spinifex-latitude-<name>` is replaced on each `up`, with the CA at `~/.spinifex/latitude-<name>-ca.pem`.
+The secret key is written to `~/.aws/credentials` (0600, other profiles kept, the old file saved beside it) and never appears in a process argument.
+
+## Token handling
+
+`LATITUDESH_AUTH_TOKEN` is read from the environment by the provider and by the script's API calls.
+It is not a Terraform variable, is on no command line (curl reads it from stdin), is not sent to a node, and is written to no file.
+`LATITUDESH_NODE_TOKEN` is checked with curl reading it from stdin and reaches each node on stdin of the install script, so it is in no argument on either side.
+`TF_LOG` is unset by the script.
+The join token that `install-node.sh` creates is shared with Vultr and OCI: its init log is root-only, and `spx admin join` still takes it as `--token` on the joining node (tracked separately).
+
+## Recovery
+
+If SSH is lost once the firewall is armed, use the serial console.
+`up` enables the server's out-of-band connection with the deployment's SSH key and prints the `ssh` command it returns before it arms the firewall, falling back to the Out-of-Band tab in the dashboard.
+Logging in there needs an OS user password, which you set yourself while SSH still works.
+A fresh deployment that cannot be recovered is cheaper to `destroy` and `up` again.
+The VLAN step prints the console command first.
+It refuses a subnet that holds the SSH peer or overlaps an existing route, and removes the VLAN again if the route to the SSH peer or the public interface changes.
+
+## destroy
+
+`destroy` asks you to type the deployment name (`--yes` skips only that), then records every ID in the Terraform state and stops without destroying anything if it cannot.
+It runs `terraform destroy`, fetches every recorded ID by itself (2xx present, 404 gone, anything else fails and keeps the records), and deletes what remains of those.
+Servers, keys, user data, virtual networks and projects whose names match the deployment but that are not recorded are listed by ID and deleted only after you type `delete-untracked` for exactly those IDs, which `--yes` never supplies; an ID that shows up later needs its own confirmation, and without a terminal they are kept and `destroy` exits 1.
+Elastic IPs are touched only by recorded ID.
+Elastic IP release is asynchronous and only works once the address is active or in error, so the sweep retries.
+Because the Elastic IP list ignores pagination, `destroy` ends by asking you to check the dashboard for leftovers and type `checked`.
+An `up` that failed while creating can leave an object nothing recorded, such as an Elastic IP; that dashboard check is the only thing that finds it.
+
+## Unverified
+
+Nothing in the three-node flow has run against real servers. For one node, only the parts the person ran have been exercised, and the script has changed since.
+
+- The OS slug `ubuntu_26_04_x64_lts` (the `latitudesh_operating_system` data source fails the plan if it is wrong).
+- That the deploy key is on `ubuntu` and `sudo -n` works for it, and that `sudo` resets nothing the script needs (the SSH peer address is read before `sudo` for that reason).
+- The power action request (`POST /servers/{id}/actions` with `power_off` and `power_on`).
+- Resolving `--project` as a slug through `GET /projects/{slug}`, and the `in_stock` shape of `GET /plans` used by the stock check, which only warns when it cannot read it.
+- The `GET` and `DELETE` paths for `/user_data/{id}` and `/virtual_networks/{id}`.
+- That the Spinifex API certificate names the node's public address, which the profile check needs.
+- That the private NIC is `eno2`, its tagging and MTU (the script uses 1500), and that the networkd files persist the VLAN across a reboot.
+- `GET /auth/api_keys` and its `token_last_slice` and `read_only` fields, which the node key check relies on; and that a key with `allowed_ips` set to the nodes' addresses still works from them.
+- The Elastic IP move itself: `PATCH /elastic_ips/{id}` with `server_id`, its 202 then `active`, the error codes, and how long a move takes; the client waits up to its move timeout and makes one write per call.
+- That the daemon reads the node token and logs "floating pool ready", then fails over when a node stops (`stop node1`) and the address lands on another node.
+- `install-node.sh` as `--user ubuntu` on latitude.sh servers.
+
+## Not exercised
+
+Any real API call from the three-node flow, `destroy`, `host-data-disk.sh` on a host and the VLAN step on a server.
+Guest launch, public reachability of the Elastic IP, the Terraform workbooks and failover are not part of `up`; prove them by hand.
+`e2e-latitude.sh` and removing the old `install-node.sh` in this directory are not done.
+
+## Tests
+
+`sh node-scripts_test.sh` checks the node token install (install, unchanged, rotation, modes, an older install converging, symlinks and owners refused; run it as root too, with `sudo sh node-scripts_test.sh`, to check that only the daemon's user can read the file) and the readiness check (old success then current failure) with stand-in tools.
+`bash deploy-argv_test.sh` checks that neither API token, nor the five characters used to find the node key, is an argument of any external command in the node token paths, and that the SSH peer address comes from the same connection after sudo's environment reset.
+`scripts/install-node_test.sh` (part of `make preflight`) checks that the join token reaches `spx admin join` only on stdin, so sudo's log never holds it.
+The first two are not part of `make preflight`.
+The Go side: `cd spinifex && GOWORK=off go test ./providers/cloud/latitude/ ./domains/network/external/floating/ ./bootstrap/config/ ./runtime/daemon/`.
@@ -0,0 +1,23 @@
+#cloud-config
+# Host preparation only. Spinifex is installed and formed over SSH by deploy-latitude.sh.
+# latitude.sh substitutes double-brace variables in user data, so this file holds none.
+write_files:
+  - path: /etc/cloud/cloud.cfg.d/99-spinifex-network.cfg
+    content: |
+      network: {config: disabled}
+  # The image's netplan lists eno2 with no address and without optional, so boot waits two minutes
+  # for it. --any returns once one link, eno1, is online; it changes the wait, not the network.
+  # UNVERIFIED: the binary path is the Ubuntu one and the drop-in first applies on the next boot.
+  - path: /etc/systemd/system/systemd-networkd-wait-online.service.d/10-spinifex-any.conf
+    content: |
+      [Service]
+      ExecStart=
+      ExecStart=/usr/lib/systemd/systemd-networkd-wait-online --any
+# runcmd carries on past a failing command, so each line stands alone.
+runcmd:
+  - [sh, -c, "command -v ufw >/dev/null && ufw --force disable || true"]
+  - [sh, -c, "systemctl disable --now unattended-upgrades.service apt-daily-upgrade.timer apt-daily.timer || true"]
+  # The image ships empty apt lists and no git, which the workbooks need.
+  - [sh, -c, "apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git || true"]
+  - [sh, -c, "systemctl daemon-reload || true"]
+  - [sh, -c, "touch /var/lib/cloud/spinifex-host-ready"]
@@ -0,0 +1,70 @@
+locals {
+  hostname_prefix = "spx-${var.deployment_name}-node"
+  project         = var.project != "" ? var.project : latitudesh_project.deploy[0].slug
+  project_id      = var.project != "" ? var.project_id : latitudesh_project.deploy[0].id
+}
+
+# Both lookups fail the plan when a slug does not exist, which is the only check of the OS slug.
+data "latitudesh_operating_system" "os" {
+  slug = var.operating_system
+}
+
+data "latitudesh_plan" "plan" {
+  slug = var.plan
+}
+
+resource "latitudesh_project" "deploy" {
+  count             = var.project == "" ? 1 : 0
+  name              = "spinifex-${var.deployment_name}"
+  description       = "Spinifex deployment ${var.deployment_name}"
+  environment       = "Development"
+  provisioning_type = "on_demand"
+}
+
+# The key is team scoped. The server login user is ubuntu, not root.
+resource "latitudesh_ssh_key" "deploy" {
+  name       = "spinifex-${var.deployment_name}"
+  public_key = trimspace(file(pathexpand(var.ssh_public_key_path)))
+}
+
+# Host preparation only. Latitude substitutes double-brace variables inside user data, so the file
+# must hold none. Spinifex is installed over SSH by deploy-latitude.sh.
+resource "latitudesh_user_data" "host" {
+  description = "spinifex-${var.deployment_name}"
+  content     = filebase64("${path.module}/cloud-init/user-data.yaml")
+}
+
+# billing must be hourly: the default is monthly, which bills upfront with no refund.
+resource "latitudesh_server" "node" {
+  count            = var.node_count
+  project          = local.project
+  hostname         = "${local.hostname_prefix}${count.index + 1}"
+  site             = var.site
+  plan             = data.latitudesh_plan.plan.slug
+  operating_system = data.latitudesh_operating_system.os.slug
+  billing          = "hourly"
+  ssh_keys         = [latitudesh_ssh_key.deploy.id]
+  user_data        = latitudesh_user_data.host.id
+
+  timeouts = {
+    create = "60m"
+  }
+}
+
+# Elastic IPs are bought here and never created or deleted by Spinifex. Each starts on a server, spread
+# over the nodes, and server_id is ignored afterwards so a move by Spinifex is not undone by a later apply.
+resource "latitudesh_elastic_ip" "pool" {
+  count = var.elastic_ip_count
+  # The provider documents ID or slug but posts the value as project_id unresolved: a slug gave 404.
+  project   = local.project_id
+  server_id = latitudesh_server.node[count.index % var.node_count].id
+
+  lifecycle {
+    ignore_changes = [server_id]
+  }
+}
+
+moved {
+  from = latitudesh_elastic_ip.node1
+  to   = latitudesh_elastic_ip.pool[0]
+}
@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+# Checks that neither API token is ever an argument of an external command in the node token
+# paths of deploy-latitude.sh. External commands are stand-ins that record their arguments and
+# their standard input; the test fails if a token is in the first and proves it is in the second.
+set -euo pipefail
+exec </dev/null
+here="$(cd "$(dirname "$0")" && pwd)"
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+mkdir "$work/bin" "$work/home"
+real_python="$(command -v python3)"
+
+AUTH=FAKEAUTHTOKEN9
+NODE=FAKENODETOKEN7
+
+# Every command the script runs outside bash records "name args" and, for the first line of
+# standard input, "name stdin: ...".
+for c in curl ssh scp aws tofu ssh-keygen; do
+    cat > "$work/bin/$c" <<'SH'
+#!/bin/sh
+echo "$(basename "$0") $*" >> "$FAKE_ARGV"
+if [ ! -t 0 ]; then
+    in="$(cat)"
+    [ -z "$in" ] || echo "$(basename "$0") stdin: $in" >> "$FAKE_STDIN"
+fi
+case "$(basename "$0")" in
+    curl) printf '{"data":[{"attributes":{"token_last_slice":"%s","read_only":false}}]}\n200' "$(printf '%s' "$FAKE_NODE" | tail -c 5)" ;;
+    ssh)
+        if [ -n "${FAKE_SSH_EVAL:-}" ]; then
+            # Behave like a login on a node: skip the options and host, run the rest with the
+            # connection variable of this connection, and let sudo reset the environment.
+            while [ "$#" -gt 0 ]; do case "$1" in ubuntu@*) shift; break ;; *) shift ;; esac; done
+            SSH_CONNECTION="$FAKE_SSH_CONNECTION" bash -c "$*"
+        else
+            echo installed
+        fi ;;
+esac
+SH
+    chmod +x "$work/bin/$c"
+done
+cat > "$work/bin/sudo" <<'SH'
+#!/bin/sh
+while [ "$#" -gt 0 ]; do case "$1" in -*) shift ;; *) break ;; esac; done
+exec env -i PATH="$PATH" "$@"
+SH
+chmod +x "$work/bin/sudo"
+cat > "$work/bin/python3" <<SH
+#!/bin/sh
+echo "python3 \$*" >> "\$FAKE_ARGV"
+exec "$real_python" "\$@"
+SH
+chmod +x "$work/bin/python3"
+
+export FAKE_ARGV="$work/argv" FAKE_STDIN="$work/stdin" FAKE_NODE="$NODE" PATH="$work/bin:$PATH" HOME="$work/home"
+: > "$FAKE_ARGV"
+: > "$FAKE_STDIN"
+export LATITUDESH_AUTH_TOKEN="$AUTH" LATITUDESH_NODE_TOKEN="$NODE"
+
+# shellcheck source=deploy-latitude.sh
+source "$here/deploy-latitude.sh"
+
+# shellcheck disable=SC2034 # read by the sourced functions
+NODES=3 IP=192.0.2.10 NODE_NAME=node1
+check_node_token >/dev/null
+install_key >/dev/null 2>&1
+
+fails=0
+for tok in "$AUTH" "$NODE" "${NODE: -5}"; do
+    if grep -q "$tok" "$FAKE_ARGV"; then
+        echo "FAIL token $tok is in a command's arguments:"
+        grep "$tok" "$FAKE_ARGV"
+        fails=$((fails + 1))
+    else
+        echo "ok   $tok is in no argument"
+    fi
+done
+for what in "curl stdin:.*Bearer $NODE" "ssh stdin: $NODE"; do
+    if grep -q "$what" "$FAKE_STDIN"; then
+        echo "ok   the check can see the token where it is sent ($what)"
+    else
+        echo "FAIL the token never reached stdin ($what): the test would pass without checking anything"
+        fails=$((fails + 1))
+    fi
+done
+
+# The peer address comes from the same connection that runs the command, whichever source
+# address that connection has, and survives sudo's environment reset.
+export FAKE_SSH_EVAL=1
+for src in 198.51.100.7 203.0.113.99; do
+    export FAKE_SSH_CONNECTION="$src 51000 192.0.2.10 22"
+    got="$(on 'printf %s "$SSH_PEER"')"
+    if [ "$got" = "$src" ]; then echo "ok   peer $src reaches the command"; else echo "FAIL peer for $src was '$got'"; fails=$((fails + 1)); fi
+done
+unset FAKE_SSH_EVAL
+
+[ "$fails" -eq 0 ]
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# host-data-disk.sh — format and mount the blank data disk at /var/lib/spinifex. Run as root ON the host.
+#
+#   host-data-disk.sh [/dev/disk/by-id/nvme-eui.<id>]
+#
+# The disk is chosen by content, never by its nvmeXn1 name, which swaps between boots. A candidate is a
+# whole-disk by-id EUI link whose device is not in the root filesystem's ancestry, has no partitions,
+# holders, swap or mounts, no filesystem signature and no partition table. Exactly one candidate must
+# remain, or the script stops and lists the disks; pass the by-id path to choose between several.
+# It never reformats: a rerun on a host already done prints the UUID and exits. The last line of stdout
+# is DATA_UUID=<uuid>, which the later install phases need.
+
+set -euo pipefail
+
+MP=/var/lib/spinifex
+LABEL=spinifex-data
+FSTAB_BAK=/etc/fstab.pre-spinifex
+MIN_BYTES="${MIN_DATA_BYTES:-100000000000}"
+WANT="${1:-}"
+
+die() { echo "STOP: $*" >&2; exit 1; }
+[ "$(id -u)" = 0 ] || die "run as root"
+
+# Resume: the mount is already there and is ours.
+if mp_uuid="$(findmnt --mountpoint "$MP" -no UUID 2>/dev/null)" && [ -n "$mp_uuid" ]; then
+    src="$(findmnt --mountpoint "$MP" -no SOURCE)"
+    [ "$(blkid -s LABEL -o value "$src")" = "$LABEL" ] || die "$MP is mounted from $src, which is not labelled $LABEL; not touching it"
+    grep -q "UUID=${mp_uuid}[[:space:]]" /etc/fstab || die "$MP is mounted but /etc/fstab has no UUID=$mp_uuid entry"
+    echo "already done: $MP is mounted from $src (UUID $mp_uuid)"
+    echo "DATA_UUID=$mp_uuid"
+    exit 0
+fi
+if grep -q "$MP" /etc/fstab; then die "/etc/fstab mentions $MP but nothing is mounted there; fix that by hand first"; fi
+[ ! -e "$MP" ] || [ -z "$(ls -A "$MP")" ] || die "$MP exists and is not empty"
+
+root_src="$(findmnt -no SOURCE /)" || die "cannot find the root source"
+root_anc="$(lsblk -s -nro NAME "$root_src")" || die "cannot read the root ancestry"
+[ -n "$root_anc" ] || die "empty root ancestry"
+
+# blank_reason ID prints why the disk is not a candidate and returns 1, or returns 0.
+blank_reason() {
+    local id="$1" dev name size kids hold sw mnt rc w
+    dev="$(readlink -e "$id")" || { echo "link does not resolve"; return 1; }
+    [ -b "$dev" ] || { echo "not a block device"; return 1; }
+    name="$(basename "$dev")"
+    if grep -qx "$name" <<<"$root_anc"; then echo "in the root ancestry"; return 1; fi
+    size="$(blockdev --getsize64 "$id")" || { echo "size unreadable"; return 1; }
+    [ "$size" -ge "$MIN_BYTES" ] || { echo "only $size bytes"; return 1; }
+    kids="$(lsblk -nro NAME "$id")" || { echo "children unreadable"; return 1; }
+    [ "$kids" = "$name" ] || { echo "has children"; return 1; }
+    hold="$(ls -A "/sys/block/$name/holders/")" || { echo "holders unreadable"; return 1; }
+    [ -z "$hold" ] || { echo "has holders"; return 1; }
+    sw="$(swapon --show=NAME --noheadings)" || { echo "swapon failed"; return 1; }
+    if grep -q "$name" <<<"$sw"; then echo "is swap"; return 1; fi
+    mnt="$(lsblk -nro MOUNTPOINTS "$id")" || { echo "mounts unreadable"; return 1; }
+    [ -z "${mnt//[[:space:]]/}" ] || { echo "is mounted"; return 1; }
+    set +e; blkid -p "$id" >/dev/null 2>&1; rc=$?; set -e
+    [ "$rc" -eq 2 ] || { echo "blkid -p says it has a signature (rc $rc)"; return 1; }
+    w="$(wipefs -n "$id")" || { echo "wipefs failed"; return 1; }
+    [ -z "$w" ] || { echo "wipefs finds signatures"; return 1; }
+    return 0
+}
+
+candidates=()
+shopt -s nullglob
+for id in /dev/disk/by-id/nvme-eui.*; do
+    case "$id" in *-part*) continue ;; esac
+    if [ -n "$WANT" ] && [ "$id" != "$WANT" ]; then continue; fi
+    if why="$(blank_reason "$id")"; then
+        candidates+=("$id")
+    else
+        echo "skip $id: $why"
+    fi
+done
+shopt -u nullglob
+
+if [ "${#candidates[@]}" -ne 1 ]; then
+    lsblk -o NAME,SIZE,TYPE,MODEL,SERIAL,FSTYPE,MOUNTPOINTS >&2 || true
+    die "expected exactly one blank data disk among the by-id EUI disks, found ${#candidates[@]}: ${candidates[*]:-none}"
+fi
+ID="${candidates[0]}"
+DEV="$(readlink -e "$ID")"
+echo "CHECKS PASSED: $ID -> $DEV"
+
+mkdir -p "$MP"
+mkfs.ext4 -q -L "$LABEL" "$ID"
+U="$(blkid -p -o value -s UUID "$ID")" || die "blkid UUID failed"
+[ -n "$U" ] || die "no UUID after mkfs"
+[ "$(blkid -p -o value -s TYPE "$ID")" = ext4 ] || die "not ext4 after mkfs"
+
+NEW="$(mktemp /etc/fstab.spinifex.XXXXXX)"
+cp /etc/fstab "$NEW"
+echo "UUID=$U $MP ext4 defaults,nofail,x-systemd.device-timeout=30s 0 2" >> "$NEW"
+findmnt --verify --tab-file "$NEW" || { rm -f "$NEW"; die "staged fstab failed verify, /etc/fstab untouched"; }
+BAK="$FSTAB_BAK.$(date +%s)"
+cp -a /etc/fstab "$BAK"
+trap 'cp -a "$BAK" /etc/fstab; systemctl daemon-reload; echo "ROLLED BACK /etc/fstab from $BAK" >&2' EXIT
+chmod 644 "$NEW"
+mv "$NEW" /etc/fstab
+systemctl daemon-reload
+mount "$MP"
+[ "$(findmnt --mountpoint "$MP" -no UUID)" = "$U" ] || die "wrong UUID at $MP"
+[ "$(findmnt --mountpoint "$MP" -no FSTYPE)" = ext4 ] || die "wrong fstype at $MP"
+[ "$(findmnt --mountpoint "$MP" -no MAJ:MIN | tr -d ' ')" = "$(lsblk -dno MAJ:MIN "$ID" | tr -d ' ')" ] || die "wrong device at $MP"
+trap - EXIT
+findmnt --mountpoint "$MP" -o TARGET,SOURCE,MAJ:MIN,UUID,FSTYPE,OPTIONS
+echo "STORAGE OK"
+echo "DATA_UUID=$U"
@@ -0,0 +1,13 @@
+# A private network needs two or more servers, so a single node has none. The VLAN carries the cluster
+# and Geneve traffic; its address is set over SSH, since the VID is only known after apply.
+resource "latitudesh_virtual_network" "lan" {
+  count       = var.node_count > 1 ? 1 : 0
+  site        = var.site
+  description = "spinifex-${var.deployment_name}"
+}
+
+resource "latitudesh_vlan_assignment" "node" {
+  count              = var.node_count > 1 ? var.node_count : 0
+  server_id          = latitudesh_server.node[count.index].id
+  virtual_network_id = latitudesh_virtual_network.lan[0].id
+}
@@ -0,0 +1,48 @@
+#!/bin/sh
+# Runs as root on a node. Reads the API token on stdin and installs it for the daemon.
+# The directory belongs to root and is not writable by anyone else, so no service can create or
+# swap an entry in it. The file belongs to the daemon's user alone (mode 0400): the other
+# services share the spinifex group, so group access would hand them the team-wide token. The
+# temporary file has an unpredictable name and is created exclusively. A symlink, or an owner
+# other than root or the daemon's user, where the directory or the file should be is refused.
+# Ownership and mode are set on every run, so an older install converges. The file's content is
+# replaced only when the token differs. Prints installed or unchanged.
+set -eu
+dir="${API_TOKEN_DIR:-/etc/spinifex/latitude}"
+root="${API_TOKEN_ROOT-root}"
+rootgroup="${API_TOKEN_ROOT_GROUP-root}"
+user="${API_TOKEN_USER-spinifex-daemon}"
+umask 077
+
+fail() { echo "$*" >&2; exit 1; }
+owner_of() { stat -c %U "$1"; }
+
+[ ! -L "$dir" ] || fail "$dir is a symlink"
+if [ ! -e "$dir" ]; then mkdir -p "$dir"; fi
+[ -d "$dir" ] || fail "$dir is not a directory"
+[ "$(owner_of "$dir")" = "$root" ] || fail "$dir is not owned by $root"
+chown "$root:$rootgroup" "$dir"
+chmod 0711 "$dir"
+
+target="$dir/api-token"
+[ ! -L "$target" ] || fail "$target is a symlink"
+if [ -e "$target" ]; then
+    [ -f "$target" ] || fail "$target is not a regular file"
+    holder="$(owner_of "$target")"
+    [ "$holder" = "$root" ] || [ "$holder" = "$user" ] || fail "$target is owned by $holder"
+fi
+
+tmp="$(mktemp "$dir/.api-token.XXXXXX")"
+trap 'rm -f "$tmp"' EXIT
+cat > "$tmp"
+[ -s "$tmp" ] || fail "empty token on stdin"
+if [ -f "$target" ] && cmp -s "$tmp" "$target"; then
+    chown "$user" "$target"
+    chmod 0400 "$target"
+    echo unchanged
+    exit 0
+fi
+chown "$user" "$tmp"
+chmod 0400 "$tmp"
+mv -f "$tmp" "$target"
+echo installed
@@ -0,0 +1,17 @@
+#!/bin/sh
+# Runs as root on a node with the pool name as $1. Reads only the journal of the daemon's
+# current invocation, so a success logged before a restart cannot hide a failure after it.
+# Prints ready, refused or waiting.
+set -eu
+pool="${1:?pool name}"
+id="$(systemctl show -p InvocationID --value spinifex-daemon)"
+[ -n "$id" ] || { echo waiting; exit 0; }
+log="$(journalctl "_SYSTEMD_INVOCATION_ID=$id" -u spinifex-daemon --no-pager -o cat 2>/dev/null || true)"
+named() { grep -F -e "pool=$pool" -e "\"pool\":\"$pool\""; }
+if printf '%s\n' "$log" | grep -F "refused the pool's first call" | named >/dev/null; then
+    echo refused
+elif printf '%s\n' "$log" | grep -F "floating pool ready" | named >/dev/null; then
+    echo ready
+else
+    echo waiting
+fi
@@ -0,0 +1,96 @@
+#!/bin/sh
+# Tests node-install-api-token.sh and node-pool-state.sh with stand-in tools.
+set -eu
+here="$(cd "$(dirname "$0")" && pwd)"
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+fails=0
+check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3' got '$2'"; fails=$((fails + 1)); fi; }
+
+# token install: A, A again, then B across a rotation; the file ends as B, owned by the daemon
+# user alone (0400) in a 0711 directory that only root may change
+me="$(id -un)"
+export API_TOKEN_DIR="$work/tok" API_TOKEN_ROOT="$me" API_TOKEN_ROOT_GROUP="$(id -gn)" API_TOKEN_USER="$me"
+install_token() { sh "$here/node-install-api-token.sh"; }
+check "first install" "$(printf 'TOKENA' | install_token)" installed
+check "same token" "$(printf 'TOKENA' | install_token)" unchanged
+check "rotation" "$(printf 'TOKENB' | install_token)" installed
+check "file holds B" "$(cat "$work/tok/api-token")" TOKENB
+check "file mode and owner" "$(stat -c '%U %a' "$work/tok/api-token")" "$me 400"
+check "dir mode and owner" "$(stat -c '%U %a' "$work/tok")" "$me 711"
+check "no temp left after install" "$(ls -A "$work/tok")" api-token
+check "empty refused" "$(printf '' | install_token 2>/dev/null || echo failed)" failed
+check "file kept after refusal" "$(cat "$work/tok/api-token")" TOKENB
+
+# an older install converges on the unchanged path too
+chmod 0640 "$work/tok/api-token"
+chmod 0755 "$work/tok"
+check "unchanged path repairs" "$(printf 'TOKENB' | install_token)" unchanged
+check "mode repaired on unchanged" "$(stat -c %a "$work/tok/api-token")" 400
+check "dir mode repaired on unchanged" "$(stat -c %a "$work/tok")" 711
+
+# who can read it: only the daemon's identity. This needs two real identities, so it runs as root only.
+if [ "$(id -u)" = 0 ]; then
+    chmod 0711 "$work"
+    export API_TOKEN_ROOT=root API_TOKEN_ROOT_GROUP=root API_TOKEN_USER=daemon
+    for mode in installed unchanged; do
+        [ "$mode" = installed ] && rm -f "$work/tok/api-token"
+        check "$mode path" "$(printf 'TOKENB' | install_token)" "$mode"
+        check "daemon reads ($mode)" "$(setpriv --reuid=daemon --regid=daemon --clear-groups cat "$work/tok/api-token")" TOKENB
+        check "another identity in the shared group cannot read ($mode)" \
+            "$(setpriv --reuid=nobody --regid=daemon --clear-groups cat "$work/tok/api-token" 2>/dev/null || echo denied)" denied
+    done
+    export API_TOKEN_ROOT="$me" API_TOKEN_ROOT_GROUP="$(id -gn)" API_TOKEN_USER="$me"
+else
+    echo "skip reading as the daemon and as another service: needs root"
+fi
+
+# a link planted where the file or the directory belongs is refused, and nothing it points at is touched
+echo SENTINEL > "$work/sentinel"
+rm -f "$work/tok/api-token"
+ln -s "$work/sentinel" "$work/tok/api-token"
+check "symlinked file refused" "$(printf 'TOKENC' | sh "$here/node-install-api-token.sh" 2>/dev/null || echo refused)" refused
+check "symlink target untouched" "$(cat "$work/sentinel")" SENTINEL
+ln -s "$work/tok" "$work/tok-link"
+check "symlinked dir refused" "$(printf 'TOKENC' | API_TOKEN_DIR="$work/tok-link" sh "$here/node-install-api-token.sh" 2>/dev/null || echo refused)" refused
+check "dir owned by someone else refused" "$(printf 'TOKENC' | API_TOKEN_ROOT=nobody-such install_token 2>/dev/null || echo refused)" refused
+rm -f "$work/tok/api-token"
+printf 'TOKENB' | install_token >/dev/null
+check "file owned by a stranger refused" "$(printf 'TOKENC' | API_TOKEN_ROOT=$me API_TOKEN_USER=nobody-such install_token 2>/dev/null || echo refused)" refused
+check "no temp left" "$(ls -A "$work/tok")" api-token
+
+# pool state: the stand-in journal serves only the lines of the invocation asked for
+mkdir "$work/bin"
+cat > "$work/bin/systemctl" <<'SH'
+#!/bin/sh
+cat "$FAKE_DIR/invocation"
+SH
+cat > "$work/bin/journalctl" <<'SH'
+#!/bin/sh
+for a in "$@"; do
+    case "$a" in _SYSTEMD_INVOCATION_ID=*) id="${a#_SYSTEMD_INVOCATION_ID=}" ;; esac
+done
+cat "$FAKE_DIR/journal.$id" 2>/dev/null
+SH
+chmod +x "$work/bin/systemctl" "$work/bin/journalctl"
+export FAKE_DIR="$work" PATH="$work/bin:$PATH"
+state() { sh "$here/node-pool-state.sh" latitude; }
+
+echo old > "$work/invocation"
+echo 'INFO floating pool ready pool=latitude source=latitude' > "$work/journal.old"
+check "current success" "$(state)" ready
+
+echo new > "$work/invocation"
+echo 'ERROR floating: the provider refused the pool'"'"'s first call; public addresses will fail until fixed pool=latitude' > "$work/journal.new"
+check "old success then current failure" "$(state)" refused
+
+echo 'INFO starting' > "$work/journal.new"
+check "current invocation not yet ready" "$(state)" waiting
+
+echo 'INFO floating pool ready pool=other source=latitude' > "$work/journal.new"
+check "another pool does not count" "$(state)" waiting
+
+echo 'INFO floating pool ready pool=latitude source=latitude' > "$work/journal.new"
+check "current success" "$(state)" ready
+
+[ "$fails" -eq 0 ]
@@ -0,0 +1,56 @@
+output "deployment_name" {
+  value = var.deployment_name
+}
+
+output "site" {
+  value = var.site
+}
+
+output "project" {
+  value = local.project
+}
+
+# provider_host_id is the server ID. public_ip is the server's own /31 address, which SSH and the
+# Spinifex API use; the Elastic IP is separate.
+output "nodes" {
+  value = [for n in latitudesh_server.node : {
+    name             = trimprefix(n.hostname, "spx-${var.deployment_name}-")
+    provider_host_id = n.id
+    public_ip        = n.primary_ipv4
+  }]
+}
+
+# The first address, for a single node's static pool.
+output "elastic_ip" {
+  value = latitudesh_elastic_ip.pool[0].address
+}
+
+# IDs as well as addresses: destroy deletes only these IDs, and the floating pool lists them.
+output "elastic_ips" {
+  value = [for e in latitudesh_elastic_ip.pool : { id = e.id, address = e.address }]
+}
+
+# The VLAN ID of the cluster network; null for one node, which has no private network.
+output "vid" {
+  value = length(latitudesh_virtual_network.lan) == 0 ? null : latitudesh_virtual_network.lan[0].vid
+}
+
+output "ssh_key_id" {
+  value = latitudesh_ssh_key.deploy.id
+}
+
+output "user_data_id" {
+  value = latitudesh_user_data.host.id
+}
+
+output "project_id" {
+  value = local.project_id
+}
+
+output "project_created" {
+  value = length(latitudesh_project.deploy) > 0
+}
+
+output "virtual_network_id" {
+  value = length(latitudesh_virtual_network.lan) == 0 ? null : latitudesh_virtual_network.lan[0].id
+}
@@ -0,0 +1,4 @@
+# The token comes from LATITUDESH_AUTH_TOKEN in the environment, which the provider reads
+# itself (its auth_token argument documents that variable). It is never a variable here, so it
+# is not written to the state or to a tfvars file. The default project is set per resource.
+provider "latitudesh" {}
@@ -0,0 +1,5 @@
+deployment_name     = "l1"
+site                = "DAL"
+plan                = "m4-metal-medium"
+node_count          = 1
+ssh_public_key_path = "~/.ssh/id_ed25519.pub"
@@ -0,0 +1,63 @@
+variable "deployment_name" {
+  type        = string
+  description = "Names every resource and the server hostnames, so two deployments in one account never collide."
+  validation {
+    # A hostname is at most 32 characters: "spx-" + name + "-node1".
+    condition     = can(regex("^[a-z0-9][a-z0-9-]{0,15}$", var.deployment_name))
+    error_message = "deployment_name must be 1-16 lowercase letters, digits or hyphens."
+  }
+}
+
+variable "project" {
+  type        = string
+  default     = ""
+  description = "ID or slug of an existing latitude.sh project to deploy into. Empty creates a project spinifex-<deployment_name> and removes it on destroy."
+}
+
+variable "project_id" {
+  type        = string
+  default     = ""
+  description = "ID of the project named by project. The Elastic IP create sends it as project_id without resolving a slug (provider 4.8.0), so it must be an ID. Unused when project is empty."
+}
+
+variable "site" {
+  type        = string
+  default     = "DAL"
+  description = "latitude.sh site slug. Pick one with stock for the plan and Elastic IP support; not SAO."
+}
+
+variable "plan" {
+  type        = string
+  default     = "m4-metal-medium"
+  description = "Server plan slug (the pricing page calls it m4.metal.medium)."
+}
+
+variable "operating_system" {
+  type        = string
+  default     = "ubuntu_26_04_x64_lts"
+  description = "OS slug. UNVERIFIED: the 26.04 slug follows the 24.04 pattern (ubuntu_24_04_x64_lts) and is checked by a data source at plan time."
+}
+
+variable "node_count" {
+  type    = number
+  default = 1
+  validation {
+    condition     = contains([1, 3], var.node_count)
+    error_message = "node_count must be 1 or 3."
+  }
+}
+
+variable "elastic_ip_count" {
+  type        = number
+  default     = 1
+  description = "Elastic IPs to buy, each 4 USD a month. Every one starts on a server and keeps it unless Spinifex moves it. latitude.sh allows 5 per team."
+  validation {
+    condition     = var.elastic_ip_count >= 1 && var.elastic_ip_count <= 5 && floor(var.elastic_ip_count) == var.elastic_ip_count
+    error_message = "elastic_ip_count must be a whole number from 1 to 5."
+  }
+}
+
+variable "ssh_public_key_path" {
+  type    = string
+  default = "~/.ssh/id_ed25519.pub"
+}
@@ -0,0 +1,7 @@
+terraform {
+  required_version = ">= 1.5.0"
+  required_providers {
+    # Pinned exactly: 4.8.0 is the version whose schema these files were written against.
+    latitudesh = { source = "latitudesh/latitudesh", version = "= 4.8.0" }
+  }
+}
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os/exec"
+	"path/filepath"
 	goruntime "runtime"
 	"syscall"
 	"time"
@@ -18,8 +19,8 @@ import (
 const sandboxID = 993

 // StartBootstrap starts the function's bootstrap at path as the sandbox uid and
-// gid with no supplementary groups, with no_new_privs, and with exactly env.
-// The caller waits on the returned command.
+// gid with no supplementary groups, with no_new_privs, and with exactly env,
+// in the bootstrap's own directory. The caller waits on the returned command.
 func StartBootstrap(ctx context.Context, path string, env []string, stdout, stderr io.Writer) (*exec.Cmd, error) {
 	cred := &syscall.Credential{Uid: sandboxID, Gid: sandboxID, Groups: []uint32{}}
 	return startBootstrap(ctx, path, env, stdout, stderr, cred)
@@ -30,6 +31,8 @@ func startBootstrap(ctx context.Context, path string, env []string, stdout, stde
 	// A nil Env inherits the agent's environment; the bootstrap gets only env.
 	cmd.Env = append([]string{}, env...)
 	cmd.Stdout, cmd.Stderr = stdout, stderr
+	// Packaged functions open their files relative to the task root.
+	cmd.Dir = filepath.Dir(path)
 	if cred != nil {
 		cmd.SysProcAttr = &syscall.SysProcAttr{Credential: cred}
 	}
@@ -169,3 +169,11 @@ func TestStartBootstrapMissingExecutableFails(t *testing.T) {
 	_, err := startBootstrap(context.Background(), filepath.Join(t.TempDir(), "bootstrap"), nil, nil, nil, nil)
 	assert.Error(t, err)
 }
+
+func TestStartBootstrapRunsInItsOwnDirectory(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "data.txt"), []byte("packaged"), 0o644))
+	bootstrap := filepath.Join(dir, "bootstrap")
+	require.NoError(t, os.WriteFile(bootstrap, []byte("#!/bin/sh\ncat data.txt\n"), 0o755))
+	assert.Equal(t, "packaged", string(run(t, bootstrap, []string{"PATH=/bin:/usr/bin"}, false)))
+}
@@ -5,6 +5,7 @@ package config
 import (
 	"fmt"
 	"log/slog"
+	"maps"
 	"net/netip"
 	"os"
 	"path/filepath"
@@ -110,7 +111,7 @@ type AWSConfig struct {
 // ExternalPool defines a range of routable IPs that Spinifex manages for public subnets.
 type ExternalPool struct {
 	Name       string   `mapstructure:"name"`        // Pool identifier (e.g., "wan", "dc1-primary")
-	Source     string   `mapstructure:"source"`      // IP source: "static" (default), "dhcp", "oci", "exoscale" or "vultr"
+	Source     string   `mapstructure:"source"`      // IP source: "static" (default), "dhcp", "oci", "exoscale", "vultr" or "latitude"
 	BindBridge string   `mapstructure:"bind_bridge"` // Linux bridge for DHCP DORA (source=dhcp only)
 	DHCPMAC    string   `mapstructure:"dhcp_mac"`    // DHCP client MAC strategy: "derived" (default) or "interface" (source=dhcp only)
 	RangeStart string   `mapstructure:"range_start"` // First IP in range (static source only)
@@ -154,6 +155,17 @@ type ExternalPool struct {
 	VultrRegion  string `mapstructure:"vultr_region"`   // Vultr region ID, e.g. "ord"; never derived from the Spinifex region
 	VultrKeyFile string `mapstructure:"vultr_key_file"` // API key file (optional; see vultr.DefaultKeyFile)
 	LabelPrefix  string `mapstructure:"label_prefix"`   // Marks this deployment's addresses, e.g. "spinifex-prod"
+
+	// Latitude names the project, token file and fixed set of Elastic IPs
+	// (source=latitude only). Each node's server ID is provider_host_id on the node.
+	LatitudeProjectID    string   `mapstructure:"latitude_project_id"`     // latitude.sh project ID the Elastic IPs belong to
+	LatitudeKeyFile      string   `mapstructure:"latitude_key_file"`       // API token file (optional; see latitude.DefaultKeyFile)
+	LatitudeElasticIPIDs []string `mapstructure:"latitude_elastic_ip_ids"` // Elastic IP IDs the pool hands out
+}
+
+// hasLatitudeKeys reports whether any latitude_* key is set.
+func (p ExternalPool) hasLatitudeKeys() bool {
+	return p.LatitudeProjectID != "" || p.LatitudeKeyFile != "" || len(p.LatitudeElasticIPIDs) > 0
 }

 // hasVultrKeys reports whether any vultr_* key, or the label prefix only a
@@ -250,7 +262,7 @@ type Config struct {

 	// ProviderHostID is this node's instance ID at its cloud provider, which a
 	// floating public address is attached to. Required on every node when a
-	// source="vultr" pool exists.
+	// source="vultr" or source="latitude" pool exists.
 	ProviderHostID string `json:"ProviderHostID,omitempty" mapstructure:"provider_host_id"`

 	Daemon      DaemonConfig      `json:"Daemon" mapstructure:"daemon"`
@@ -355,6 +367,11 @@ type EBSConfig struct {
 	CacheSizeMB *int `json:"CacheSizeMB" mapstructure:"cache_size_mb"`
 }

+// SupportedEBSProvider reports whether name is a provider a node may run.
+func SupportedEBSProvider(name string) bool {
+	return name == EBSProviderViperblockd || name == EBSProviderQEMUNBD
+}
+
 // ResolvedProvider normalizes an empty Provider to EBSProviderViperblockd,
 // the default provider.
 func (c EBSConfig) ResolvedProvider() string {
@@ -801,6 +818,67 @@ func validateVultrPool(cc *ClusterConfig, p ExternalPool) error {
 	return nil
 }

+// validateLatitudePool checks a source="latitude" pool. The Elastic IPs are
+// bought ahead of time and named here, the API picks nothing, so the range,
+// gateway and label keys mean nothing, and every node must name its server.
+func validateLatitudePool(cc *ClusterConfig, p ExternalPool) error {
+	if p.LatitudeProjectID == "" || len(p.LatitudeElasticIPIDs) == 0 {
+		return fmt.Errorf("config: [[network.external_pools]] %q: source=\"latitude\" requires latitude_project_id and latitude_elastic_ip_ids", p.Name)
+	}
+	seen := map[string]struct{}{}
+	for _, id := range p.LatitudeElasticIPIDs {
+		if id == "" {
+			return fmt.Errorf("config: [[network.external_pools]] %q: latitude_elastic_ip_ids has an empty entry", p.Name)
+		}
+		if _, dup := seen[id]; dup {
+			return fmt.Errorf("config: [[network.external_pools]] %q: latitude_elastic_ip_ids lists %q twice", p.Name, id)
+		}
+		seen[id] = struct{}{}
+	}
+	if p.LatitudeKeyFile != "" && !filepath.IsAbs(p.LatitudeKeyFile) {
+		return fmt.Errorf("config: [[network.external_pools]] %q: latitude_key_file must be an absolute path, got %q", p.Name, p.LatitudeKeyFile)
+	}
+	if p.BindBridge != "" || p.DHCPMAC != "" {
+		return fmt.Errorf("config: [[network.external_pools]] %q: bind_bridge/dhcp_mac are only valid with source=\"dhcp\"", p.Name)
+	}
+	if p.RangeStart != "" || p.RangeEnd != "" || p.Gateway != "" || p.GatewayIP != "" {
+		return fmt.Errorf("config: [[network.external_pools]] %q: range_start/range_end/gateway/gateway_ip not allowed with source=\"latitude\" (the Elastic IPs are listed by ID)", p.Name)
+	}
+	if p.GwLrpRangeStart != "" || p.GwLrpRangeEnd != "" {
+		return fmt.Errorf("config: [[network.external_pools]] %q: gw_lrp_range_start/gw_lrp_range_end not allowed with source=\"latitude\" (a gateway address per VPC would use up the Elastic IP limit)", p.Name)
+	}
+	if cc.Network.ExternalMode != "nat" {
+		return fmt.Errorf("config: [[network.external_pools]] %q: source=\"latitude\" requires [network] external_mode = \"nat\" (routed); a routed Elastic IP is delivered to the server's own MAC", p.Name)
+	}
+	for name, n := range cc.Nodes {
+		if n.ProviderHostID == "" {
+			return fmt.Errorf("config: [nodes.%s] provider_host_id is required with a source=\"latitude\" pool (%q): it is the latitude.sh server ID the node's Elastic IPs move to", name, p.Name)
+		}
+	}
+	return nil
+}
+
+// validateSingleEBSProvider refuses nodes whose explicit [nodes.<name>.ebs]
+// providers differ. A peer stanza without one is unknown, not the default: the
+// generated config omits it, and the cluster's KV lock is the authoritative check.
+func validateSingleEBSProvider(cc *ClusterConfig) error {
+	var first, want string
+	for _, name := range slices.Sorted(maps.Keys(cc.Nodes)) {
+		if !viper.IsSet("nodes." + name + ".ebs.provider") {
+			continue
+		}
+		got := cc.Nodes[name].EBS.ResolvedProvider()
+		if first == "" {
+			first, want = name, got
+			continue
+		}
+		if got != want {
+			return fmt.Errorf("config: [nodes.%s.ebs] provider resolves to %q but [nodes.%s.ebs] resolves to %q; every node in a cluster must use the same EBS provider", first, want, name, got)
+		}
+	}
+	return nil
+}
+
 func validateClusterConfig(cc *ClusterConfig) error {
 	if viper.IsSet("network.external_dhcp") {
 		return fmt.Errorf("config: [network] external_dhcp is no longer supported; remove the key (static WAN-pool allocation only)")
@@ -809,9 +887,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		if viper.IsSet("nodes." + nodeName + ".vpcd.dhcp_bind_bridge") {
 			return fmt.Errorf("config: [nodes.%s.vpcd] dhcp_bind_bridge is no longer supported; remove the key (vpcd no longer runs a DHCP client)", nodeName)
 		}
-		switch nodeCfg.EBS.Provider {
-		case "", EBSProviderViperblockd, EBSProviderQEMUNBD:
-		case EBSProviderEmbedded:
+		switch {
+		case nodeCfg.EBS.Provider == "", SupportedEBSProvider(nodeCfg.EBS.Provider):
+		case nodeCfg.EBS.Provider == EBSProviderEmbedded:
 			// Refuse rather than silently upgrade: the embedded engine is gone,
 			// and starting anyway would migrate this node's volumes to
 			// ebsmetadata one-way without the operator ever asking for it.
@@ -829,6 +907,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 			return fmt.Errorf("config: [nodes.%s.ebs] cache_size_mb=%d must not be negative; use 0 to disable the cache, or remove the key for the default of %d", nodeName, *c, DefaultCacheSizeMB)
 		}
 	}
+	if err := validateSingleEBSProvider(cc); err != nil {
+		return err
+	}

 	if err := validateNorthstarCluster(cc); err != nil {
 		return err
@@ -849,6 +930,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		end   netip.Addr
 	}
 	var ranges []poolRange
+	// latitudeIDs maps each Elastic IP ID to the pool naming it. Every pool keeps its own
+	// record of what it handed out, so an ID in two pools would be given to two guests.
+	latitudeIDs := map[string]string{}
 	for _, p := range cc.Network.ExternalPools {
 		if p.Source != "oci" && (p.OCICompartmentID != "" || p.OCIVNICID != "" || p.OCIVNICIface != "" || p.OCISubnetID != "" || p.OCIPublicIPPool != "" || p.OCIConfigFile != "" || p.OCIConfigProfile != "") {
 			return fmt.Errorf("config: [[network.external_pools]] %q: oci_* keys are only valid with source=\"oci\"", p.Name)
@@ -856,6 +940,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		if p.Source != "exoscale" && p.hasExoscaleKeys() {
 			return fmt.Errorf("config: [[network.external_pools]] %q: exoscale_* keys are only valid with source=\"exoscale\"", p.Name)
 		}
+		if p.Source != "latitude" && p.hasLatitudeKeys() {
+			return fmt.Errorf("config: [[network.external_pools]] %q: latitude_* keys are only valid with source=\"latitude\"", p.Name)
+		}
 		if p.Source != "vultr" && p.hasVultrKeys() {
 			return fmt.Errorf("config: [[network.external_pools]] %q: vultr_* and label_prefix keys are only valid with source=\"vultr\"", p.Name)
 		}
@@ -955,8 +1042,19 @@ func validateClusterConfig(cc *ClusterConfig) error {
 				return err
 			}
 			continue
+		case "latitude":
+			if err := validateLatitudePool(cc, p); err != nil {
+				return err
+			}
+			for _, id := range p.LatitudeElasticIPIDs {
+				if other, dup := latitudeIDs[id]; dup {
+					return fmt.Errorf("config: [[network.external_pools]] %q: Elastic IP %q is also named by pool %q; each Elastic IP may belong to one pool", p.Name, id, other)
+				}
+				latitudeIDs[id] = p.Name
+			}
+			continue
 		default:
-			return fmt.Errorf("config: [[network.external_pools]] %q: source=%q unsupported; use \"static\", \"dhcp\", \"oci\", \"exoscale\" or \"vultr\"", p.Name, p.Source)
+			return fmt.Errorf("config: [[network.external_pools]] %q: source=%q unsupported; use \"static\", \"dhcp\", \"oci\", \"exoscale\", \"vultr\" or \"latitude\"", p.Name, p.Source)
 		}
 		if p.RangeStart == "" || p.RangeEnd == "" {
 			continue
@@ -1108,6 +1108,45 @@ provider = "bogus"
 	assert.Contains(t, err.Error(), `provider="bogus"`)
 }

+func TestLoadConfig_EBSProvider_SingleProviderPerCluster(t *testing.T) {
+	stanza := func(node, provider string) string {
+		return fmt.Sprintf("[nodes.%s.ebs]\nprovider = %q\n", node, provider)
+	}
+	cases := []struct {
+		name    string
+		ebs     []string
+		wantErr string
+	}{
+		{"all unset", []string{"", "", ""}, ""},
+		{"explicit viperblockd beside unset peers", []string{"", stanza("n2", EBSProviderViperblockd), ""}, ""},
+		{"explicit qemunbd beside unset peers", []string{stanza("n1", EBSProviderQEMUNBD), "", ""}, ""},
+		{"all explicit qemunbd", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderQEMUNBD), stanza("n3", EBSProviderQEMUNBD)}, ""},
+		{"two explicit providers differ", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderViperblockd), ""},
+			`[nodes.n1.ebs] provider resolves to "qemunbd" but [nodes.n2.ebs] resolves to "viperblockd"`},
+		{"only the third node differs", []string{stanza("n1", EBSProviderViperblockd), stanza("n2", EBSProviderViperblockd), stanza("n3", EBSProviderQEMUNBD)},
+			`[nodes.n1.ebs] provider resolves to "viperblockd" but [nodes.n3.ebs] resolves to "qemunbd"`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			body := "node = \"n1\"\n"
+			for i, ebs := range tc.ebs {
+				body += fmt.Sprintf("\n[nodes.n%d]\nregion = \"us-east-1\"\n%s", i+1, ebs)
+			}
+			resetViper(t)
+			path := filepath.Join(t.TempDir(), "spinifex.toml")
+			require.NoError(t, os.WriteFile(path, []byte(body), 0600))
+
+			_, err := LoadConfig(path)
+			if tc.wantErr == "" {
+				require.NoError(t, err)
+				return
+			}
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.wantErr)
+		})
+	}
+}
+
 func TestEBSConfig_ResolvedTunablesDefault(t *testing.T) {
 	var c EBSConfig
 	assert.Equal(t, DefaultNBDKitThreads, c.ResolvedThreads())
@@ -1511,6 +1550,65 @@ func TestLoadConfig_NetworkPoolVultrRejections(t *testing.T) {
 	}
 }

+const latitudePoolBase = `
+source = "latitude"
+latitude_project_id = "proj_1"
+latitude_elastic_ip_ids = ["eip_1", "eip_2"]
+`
+
+func TestLoadConfig_NetworkPoolLatitudeSourceAccepted(t *testing.T) {
+	cfg, err := loadVultrPool(t, "nat", latitudePoolBase+"latitude_key_file = \"/etc/spinifex/latitude/api-token\"\n",
+		[2]string{"sv_a", "sv_b"})
+	require.NoError(t, err)
+	p := cfg.Network.ExternalPools[0]
+	assert.Equal(t, "latitude", p.Source)
+	assert.Equal(t, "proj_1", p.LatitudeProjectID)
+	assert.Equal(t, []string{"eip_1", "eip_2"}, p.LatitudeElasticIPIDs)
+	assert.Equal(t, "/etc/spinifex/latitude/api-token", p.LatitudeKeyFile)
+	assert.Equal(t, "sv_a", cfg.Nodes["n1"].ProviderHostID)
+}
+
+func TestLoadConfig_NetworkPoolLatitudeRejections(t *testing.T) {
+	both := [2]string{"a", "b"}
+	cases := []struct {
+		name, mode, body string
+		nodes            [2]string
+		want             string
+	}{
+		{"pool mode", "pool", latitudePoolBase, both, `requires [network] external_mode = "nat"`},
+		{"no project", "nat", "source = \"latitude\"\nlatitude_elastic_ip_ids = [\"e\"]\n", both, "requires latitude_project_id and latitude_elastic_ip_ids"},
+		{"no ids", "nat", "source = \"latitude\"\nlatitude_project_id = \"p\"\n", both, "requires latitude_project_id and latitude_elastic_ip_ids"},
+		{"duplicate id", "nat", "source = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"e\", \"e\"]\n", both, "twice"},
+		{"empty id", "nat", "source = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"\"]\n", both, "empty entry"},
+		{"relative key file", "nat", latitudePoolBase + "latitude_key_file = \"token\"\n", both, "absolute path"},
+		{"label prefix", "nat", latitudePoolBase + "label_prefix = \"x\"\n", both, "only valid with source=\"vultr\""},
+		{"range", "nat", latitudePoolBase + "range_start = \"1.2.3.4\"\nrange_end = \"1.2.3.4\"\n", both, "listed by ID"},
+		{"a node without its server id", "nat", latitudePoolBase, [2]string{"a", ""}, "[nodes.n2] provider_host_id is required"},
+		{"latitude keys on static", "nat", "latitude_project_id = \"p\"\n", both, `only valid with source="latitude"`},
+		{"vultr keys on latitude", "nat", latitudePoolBase + "vultr_region = \"ord\"\n", both, `only valid with source="vultr"`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := loadVultrPool(t, tc.mode, tc.body, tc.nodes)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.want)
+		})
+	}
+}
+
+func TestLoadConfig_NetworkPoolLatitudeIDInTwoPoolsIsRejected(t *testing.T) {
+	resetViper(t)
+	body := "node = \"n1\"\n\n[network]\nexternal_mode = \"nat\"\n\n" +
+		"[[network.external_pools]]\nname = \"a\"\nsource = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"e1\", \"e2\"]\n\n" +
+		"[[network.external_pools]]\nname = \"b\"\nsource = \"latitude\"\nlatitude_project_id = \"p\"\nlatitude_elastic_ip_ids = [\"e3\", \"e2\"]\n\n" +
+		"[nodes.n1]\nregion = \"us-east-1\"\nprovider_host_id = \"sv_1\"\n"
+	path := filepath.Join(t.TempDir(), "spinifex.toml")
+	require.NoError(t, os.WriteFile(path, []byte(body), 0600))
+	_, err := LoadConfig(path)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), `"e2" is also named by pool "a"`)
+}
+
 func TestLoadConfig_NorthstarNameservers(t *testing.T) {
 	load := func(t *testing.T, block string) (*ClusterConfig, error) {
 		t.Helper()
@@ -69,6 +69,28 @@ var (
 	_ TagIndexOwner       = (*ec2tags.TagsServiceImpl)(nil)
 )

+// kindPrefixes maps each cleanable kind to its resource-ID prefix in the tag index.
+var kindPrefixes = map[string]string{
+	ec2v1.PlatformKindKeyPair:        keyPairIDPrefix,
+	ec2v1.PlatformKindPlacementGroup: placementGroupIDPrefix,
+	ec2v1.PlatformKindLaunchTemplate: launchTemplateIDPrefix,
+}
+
+// selectKinds returns the kinds a request cleans, in report order. No kinds
+// means all three; a kind that is not cleanable, such as the tag index, is refused.
+func selectKinds(requested []string) ([]string, error) {
+	all := []string{ec2v1.PlatformKindKeyPair, ec2v1.PlatformKindPlacementGroup, ec2v1.PlatformKindLaunchTemplate}
+	if len(requested) == 0 {
+		return all, nil
+	}
+	for _, kind := range requested {
+		if !slices.Contains(all, kind) {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The kind is not one of key-pair, placement-group or launch-template.")
+		}
+	}
+	return slices.DeleteFunc(all, func(kind string) bool { return !slices.Contains(requested, kind) }), nil
+}
+
 // DefaultSettleWindow is how old headerless residue must be before it is
 // removed; anything younger may be a create still in flight.
 const DefaultSettleWindow = 10 * time.Minute
@@ -102,6 +124,7 @@ const (
 // CleanupPlatform deletes the account's key pairs, empty placement groups and
 // launch templates, removes settled residue and orphaned tag-index entries,
 // then re-scans every owner store to build the result. It is safe to re-run.
+// Request.Kinds limits the run to those kinds and their tag-index entries.
 func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, error) {
 	if req == nil || req.AccountID == "" {
 		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "The request must specify an account ID.")
@@ -118,27 +141,54 @@ func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, e
 	acct := req.AccountID
 	res := &Result{AccountID: acct, CorrelationRef: req.CorrelationRef}

-	refs := c.readReferences(ctx, acct)
-	keyRun := c.cleanKeyPairs(ctx, acct, settledBefore, refs)
-	groupRun := c.cleanPlacementGroups(ctx, acct, refs)
-	templateRun := c.cleanLaunchTemplates(ctx, acct, settledBefore)
+	kinds, err := selectKinds(req.Kinds)
+	if err != nil {
+		return nil, err
+	}
+	var prefixes []string
+	for _, kind := range kinds {
+		prefixes = append(prefixes, kindPrefixes[kind])
+	}
+
+	var keyRun *keyPairRun
+	var groupRun *placementGroupRun
+	var templateRun *launchTemplateRun
+	if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) || slices.Contains(kinds, ec2v1.PlatformKindPlacementGroup) {
+		refs := c.readReferences(ctx, acct)
+		if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) {
+			keyRun = c.cleanKeyPairs(ctx, acct, settledBefore, refs)
+		}
+		if slices.Contains(kinds, ec2v1.PlatformKindPlacementGroup) {
+			groupRun = c.cleanPlacementGroups(ctx, acct, refs)
+		}
+	}
+	if slices.Contains(kinds, ec2v1.PlatformKindLaunchTemplate) {
+		templateRun = c.cleanLaunchTemplates(ctx, acct, settledBefore)
+	}

 	// Tags are listed before the owner re-scans, so a resource created in
 	// between is seen as owned and keeps its tags.
 	tagKind := c.startKind(ec2v1.PlatformKindTagIndex)
-	tagged, tagErr := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	tagged, tagErr := c.tags.ScanAccountResourceIDs(ctx, acct, prefixes...)
 	if tagErr != nil {
 		slog.WarnContext(ctx, "Platform cleanup: tag index listing failed", "err", tagErr)
 		tagKind.incomplete(ec2v1.PlatformReasonStoreListFailed)
 	}

-	owners := map[string]ownedIDs{
-		keyPairIDPrefix:        c.finishKeyPairs(ctx, acct, keyRun, res),
-		placementGroupIDPrefix: c.finishPlacementGroups(ctx, acct, groupRun, res),
-		launchTemplateIDPrefix: c.finishLaunchTemplates(ctx, acct, templateRun, res),
+	owners := map[string]ownedIDs{}
+	if keyRun != nil {
+		owners[keyPairIDPrefix] = c.finishKeyPairs(ctx, acct, keyRun, res)
+		res.Kinds = append(res.Kinds, keyRun.kind.report)
 	}
-	res.Kinds = []KindReport{keyRun.kind.report, groupRun.kind.report, templateRun.kind.report}
-	res.Kinds = append(res.Kinds, c.cleanTagIndex(ctx, acct, tagKind, tagged, tagErr == nil, owners, res))
+	if groupRun != nil {
+		owners[placementGroupIDPrefix] = c.finishPlacementGroups(ctx, acct, groupRun, res)
+		res.Kinds = append(res.Kinds, groupRun.kind.report)
+	}
+	if templateRun != nil {
+		owners[launchTemplateIDPrefix] = c.finishLaunchTemplates(ctx, acct, templateRun, res)
+		res.Kinds = append(res.Kinds, templateRun.kind.report)
+	}
+	res.Kinds = append(res.Kinds, c.cleanTagIndex(ctx, acct, tagKind, tagged, prefixes, tagErr == nil, owners, res))

 	slog.InfoContext(ctx, "Platform cleanup completed", "accountID", acct, "correlationRef", req.CorrelationRef,
 		"items", len(res.Items), "duration_ms", otelsetup.Millis(time.Since(start)))
@@ -775,3 +775,38 @@ func (f failingResidue) CleanupAccountResidue(ctx context.Context, acct string,
 	}
 	return &ec2launchtemplate.AccountCleanup{KeptTemplateIDs: scan.LiveTemplateIDs, OrphanBodiesFailed: scan.OrphanVersionBodies}, nil
 }
+
+// A run limited to one kind cleans that kind and its tag-index entries only:
+// the other kinds keep their resources and their tags, and are not reported.
+func TestCleanupPlatformLimitedToKindsLeavesTheRestAlone(t *testing.T) {
+	o := newOwners(t)
+	keyID := o.importKey(t, testAccount, "k1")
+	groupID := o.createGroup(t, testAccount, "g1")
+	o.tag(t, testAccount, keyID)
+
+	c := o.newCleanup()
+	c.now = func() time.Time { return time.Now().Add(time.Hour) }
+	res, err := c.CleanupPlatform(t.Context(), &Request{AccountID: testAccount, Kinds: []string{ec2v1.PlatformKindPlacementGroup}})
+	require.NoError(t, err)
+
+	assert.Equal(t, []Item{item(groupID, ec2v1.PlatformKindPlacementGroup, DispositionDeleted, "")}, res.Items)
+	assert.Equal(t, map[string]string{
+		ec2v1.PlatformKindPlacementGroup: ec2v1.PlatformStatusObservedClean,
+		ec2v1.PlatformKindTagIndex:       ec2v1.PlatformStatusObservedClean,
+	}, statuses(res))
+
+	keys, err := o.keys.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, keys.LiveKeyPairIDs, "an unselected kind is not touched")
+	tagged, err := o.tags.ScanAccountResourceIDs(t.Context(), testAccount, "key-")
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, tagged, "an unselected kind keeps its tags")
+}
+
+func TestCleanupPlatformRefusesAKindItCannotClean(t *testing.T) {
+	o := newOwners(t)
+	for _, kinds := range [][]string{{ec2v1.PlatformKindTagIndex}, {"volume"}} {
+		_, err := o.newCleanup().CleanupPlatform(t.Context(), &Request{AccountID: testAccount, Kinds: kinds})
+		require.Error(t, err, "%v", kinds)
+	}
+}
@@ -282,15 +282,15 @@ func (c *Cleanup) finishLaunchTemplates(ctx context.Context, acct string, run *l
 // cleanTagIndex clears entries whose owner re-scan resolved and holds no
 // record for the ID, then re-lists the index to prove each clear. Entries of
 // resources that still exist are theirs, not residue, and are left alone.
-func (c *Cleanup) cleanTagIndex(ctx context.Context, acct string, kind *kindRun, tagged []string, listed bool, owners map[string]ownedIDs, res *Result) KindReport {
+func (c *Cleanup) cleanTagIndex(ctx context.Context, acct string, kind *kindRun, tagged, prefixes []string, listed bool, owners map[string]ownedIDs, res *Result) KindReport {
 	if listed {
-		c.clearOrphanTags(ctx, acct, kind, tagged, owners, res)
+		c.clearOrphanTags(ctx, acct, kind, tagged, prefixes, owners, res)
 	}
 	c.finishKind(kind)
 	return kind.report
 }

-func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRun, tagged []string, owners map[string]ownedIDs, res *Result) {
+func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRun, tagged, prefixes []string, owners map[string]ownedIDs, res *Result) {
 	// A resource this run reports as not deleted owns its tags even when its
 	// kind's re-scan failed.
 	reported := map[string]bool{}
@@ -317,7 +317,7 @@ func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRu
 			cleared = append(cleared, id)
 		}
 	}
-	after, err := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	after, err := c.tags.ScanAccountResourceIDs(ctx, acct, prefixes...)
 	if err != nil {
 		slog.WarnContext(ctx, "Platform cleanup: tag index re-scan failed", "err", err)
 		kind.incomplete(ec2v1.PlatformReasonStoreListFailed)
@@ -7,6 +7,9 @@ import "time"
 type Request struct {
 	AccountID      string `json:"account_id"`
 	CorrelationRef string `json:"correlation_ref,omitempty"`
+
+	// Kinds limits the run to these platform kinds. Empty means all three.
+	Kinds []string `json:"kinds,omitempty"`
 }

 // Result is what the owner stores hold after one run. Kinds reuse the ec2.v1
@@ -1,80 +0,0 @@
-package instance
-
-import (
-	"context"
-	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/nats-io/nats.go"
-)
-
-// NATSInstanceService handles instance operations via NATS. Fan-out methods
-// (DescribeInstances, DescribeInstanceTypes) intentionally error — use the
-// gateway's scatter-gather helper for cluster-wide results.
-type NATSInstanceService struct {
-	natsConn *nats.Conn
-}
-
-var _ InstanceService = (*NATSInstanceService)(nil)
-
-// NewNATSInstanceService creates a new NATS-based instance service.
-func NewNATSInstanceService(conn *nats.Conn) InstanceService {
-	return &NATSInstanceService{natsConn: conn}
-}
-
-func (s *NATSInstanceService) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
-	if input == nil || input.InstanceType == nil {
-		return nil, fmt.Errorf("instance type is required")
-	}
-	topic := fmt.Sprintf("ec2.RunInstances.%s", aws.StringValue(input.InstanceType))
-	return natsmsg.NATSRequest[ec2.Reservation](ctx, s.natsConn, topic, input, 5*time.Minute, accountID)
-}
-
-// DescribeInstances is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstances(_ context.Context, _ *ec2.DescribeInstancesInput, _ string) (*ec2.DescribeInstancesOutput, error) {
-	return nil, fmt.Errorf("DescribeInstances is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-// DescribeInstanceTypes is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstanceTypes(_ context.Context, _ *ec2.DescribeInstanceTypesInput, _ string) (*DescribeInstanceTypesNodeOutput, error) {
-	return nil, fmt.Errorf("DescribeInstanceTypes is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-// DescribeInstanceStatus is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstanceStatus(_ context.Context, _ *ec2.DescribeInstanceStatusInput, _ string) (*ec2.DescribeInstanceStatusOutput, error) {
-	return nil, fmt.Errorf("DescribeInstanceStatus is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-func (s *NATSInstanceService) DescribeInstanceAttribute(ctx context.Context, input *ec2.DescribeInstanceAttributeInput, accountID string) (*ec2.DescribeInstanceAttributeOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstanceAttributeOutput](ctx, s.natsConn, "ec2.DescribeInstanceAttribute", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) DescribeStoppedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstancesOutput](ctx, s.natsConn, "ec2.DescribeStoppedInstances", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) DescribeTerminatedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstancesOutput](ctx, s.natsConn, "ec2.DescribeTerminatedInstances", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) ModifyInstanceAttribute(ctx context.Context, input *ec2.ModifyInstanceAttributeInput, accountID string) (*ec2.ModifyInstanceAttributeOutput, error) {
-	return natsmsg.NATSRequest[ec2.ModifyInstanceAttributeOutput](ctx, s.natsConn, "ec2.ModifyInstanceAttribute", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) ModifyInstanceMetadataOptions(ctx context.Context, input *ec2.ModifyInstanceMetadataOptionsInput, accountID string) (*ec2.ModifyInstanceMetadataOptionsOutput, error) {
-	return natsmsg.NATSRequest[ec2.ModifyInstanceMetadataOptionsOutput](ctx, s.natsConn, "ec2.ModifyInstanceMetadataOptions", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) StartStoppedInstance(ctx context.Context, input *StartStoppedInstanceInput, accountID string) (*StartStoppedInstanceOutput, error) {
-	return natsmsg.NATSRequest[StartStoppedInstanceOutput](ctx, s.natsConn, "ec2.StartStoppedInstance", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) TerminateStoppedInstance(ctx context.Context, input *TerminateStoppedInstanceInput, accountID string) (*TerminateStoppedInstanceOutput, error) {
-	return natsmsg.NATSRequest[TerminateStoppedInstanceOutput](ctx, s.natsConn, "ec2.TerminateStoppedInstance", input, 30*time.Second, accountID)
-}
@@ -1,364 +0,0 @@
-package instance
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/nats-io/nats-server/v2/server"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// startTestNATSServer starts an embedded NATS server for testing.
-func startTestNATSServer(t *testing.T) (*server.Server, string) {
-	t.Helper()
-	ns, nc := testutil.StartTestNATS(t)
-	_ = nc // connection managed by testutil cleanup
-	return ns, ns.ClientURL()
-}
-
-// createValidRunInstancesInput creates a valid RunInstancesInput for testing.
-func createValidRunInstancesInput() *ec2.RunInstancesInput {
-	return &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String("t3.micro"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-		KeyName:      aws.String("test-key-pair"),
-		SecurityGroupIds: []*string{
-			aws.String("sg-0123456789abcdef0"),
-		},
-		SubnetId: aws.String("subnet-6e7f829e"),
-		UserData: aws.String("#!/bin/bash\necho 'test'"),
-	}
-}
-
-// createValidReservation creates a valid ec2.Reservation response for testing.
-func createValidReservation() *ec2.Reservation {
-	return &ec2.Reservation{
-		ReservationId: aws.String("r-0123456789abcdef0"),
-		OwnerId:       aws.String("123456789012"),
-		Instances: []*ec2.Instance{
-			{
-				InstanceId:   aws.String("i-0123456789abcdef0"),
-				InstanceType: aws.String("t3.micro"),
-				ImageId:      aws.String("ami-0abcdef1234567890"),
-				State: &ec2.InstanceState{
-					Code: aws.Int64(0),
-					Name: aws.String("pending"),
-				},
-				PrivateIpAddress: aws.String("10.0.1.100"),
-				SubnetId:         aws.String("subnet-6e7f829e"),
-			},
-		},
-	}
-}
-
-// TestNATSInstanceService_RunInstances_Success tests successful RunInstances operation.
-func TestNATSInstanceService_RunInstances_Success(t *testing.T) {
-	// Skip if LOG_IGNORE is set
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	// Start test NATS server
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	// Create NATS connections
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err, "Failed to connect to NATS")
-	defer nc.Close()
-
-	// Create mock daemon subscriber that responds with valid reservation
-	// Subscribe to the per-instance-type topic (t3.micro from createValidRunInstancesInput)
-	mockReservation := createValidReservation()
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		// Validate that request is properly formatted
-		var input ec2.RunInstancesInput
-		err := json.Unmarshal(msg.Data, &input)
-		if err != nil {
-			t.Errorf("Mock daemon received invalid JSON: %v", err)
-			return
-		}
-
-		// Send back valid reservation
-		responseData, _ := json.Marshal(mockReservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err, "Failed to subscribe mock daemon")
-
-	// Create NATSInstanceService
-	service := NewNATSInstanceService(nc)
-	require.NotNil(t, service)
-
-	// Create valid input
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify success
-	require.NoError(t, err, "RunInstances should succeed")
-	require.NotNil(t, reservation, "Reservation should not be nil")
-
-	// Verify reservation contents
-	assert.Equal(t, *mockReservation.ReservationId, *reservation.ReservationId)
-	assert.Equal(t, *mockReservation.OwnerId, *reservation.OwnerId)
-	require.Len(t, reservation.Instances, 1)
-
-	instance := reservation.Instances[0]
-	assert.Equal(t, "i-0123456789abcdef0", *instance.InstanceId)
-	assert.Equal(t, "t3.micro", *instance.InstanceType)
-	assert.Equal(t, int64(0), *instance.State.Code)
-	assert.Equal(t, "pending", *instance.State.Name)
-}
-
-// TestNATSInstanceService_RunInstances_DaemonError tests error handling when daemon returns error.
-func TestNATSInstanceService_RunInstances_DaemonError(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that responds with error
-	// Subscribe to the per-instance-type topic for the requested type
-	_, err = nc.QueueSubscribe("ec2.RunInstances.invalid.type", "spinifex-workers", func(msg *nats.Msg) {
-		// Send back error response
-		errorResponse := awserrors.GenerateErrorPayload("InvalidInstanceType")
-		msg.Respond(errorResponse)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-
-	input := &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String("invalid.type"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-	}
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify error handling - error should be just the AWS error code for gateway lookup
-	require.Error(t, err, "Should return error when daemon returns error")
-	assert.Nil(t, reservation, "Reservation should be nil on error")
-	assert.Equal(t, "InvalidInstanceType", err.Error(), "Error should be the AWS error code")
-}
-
-// TestNATSInstanceService_RunInstances_NoSubscriber tests behavior when no daemon is subscribed.
-func TestNATSInstanceService_RunInstances_NoSubscriber(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// No subscriber - no daemon running
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	start := time.Now()
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-	duration := time.Since(start)
-
-	// Verify error behavior
-	// When no subscribers are available, NATS returns ErrNoResponders immediately
-	require.Error(t, err, "Should fail when no subscriber")
-	assert.Nil(t, reservation, "Reservation should be nil")
-	assert.ErrorIs(t, err, nats.ErrNoResponders, "Error should be ErrNoResponders")
-
-	// Should fail quickly (within 1 second), not wait for 30 second timeout
-	assert.Less(t, duration.Seconds(), 1.0, "Should fail immediately when no responders")
-}
-
-// TestNATSInstanceService_RunInstances_InvalidResponse tests handling of malformed responses.
-func TestNATSInstanceService_RunInstances_InvalidResponse(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that responds with invalid JSON
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		// Send back malformed JSON
-		msg.Respond([]byte(`{"invalid": json response`))
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify error handling
-	require.Error(t, err, "Should return error for invalid JSON")
-	assert.Nil(t, reservation, "Reservation should be nil")
-	assert.Contains(t, err.Error(), "failed to unmarshal", "Error should indicate unmarshal failure")
-}
-
-// TestNATSInstanceService_RunInstances_MarshalError tests handling when input marshaling fails.
-func TestNATSInstanceService_RunInstances_MarshalError(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	service := NewNATSInstanceService(nc)
-
-	// Nil input should be rejected before hitting NATS
-	reservation, err := service.RunInstances(context.Background(), nil, "123456789012")
-	require.Error(t, err, "Should handle nil input")
-	assert.Nil(t, reservation)
-	assert.Contains(t, err.Error(), "instance type is required")
-}
-
-// TestNATSInstanceService_RunInstances_MultipleInstances tests launching multiple instances.
-func TestNATSInstanceService_RunInstances_MultipleInstances(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that returns multiple instances
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		var input ec2.RunInstancesInput
-		json.Unmarshal(msg.Data, &input)
-
-		// Create reservation with count matching MinCount
-		count := int(*input.MinCount)
-		instances := make([]*ec2.Instance, count)
-		for i := range count {
-			instanceId := aws.String(aws.StringValue(aws.String(fmt.Sprintf("i-%d", i))))
-			instances[i] = &ec2.Instance{
-				InstanceId:   instanceId,
-				InstanceType: input.InstanceType,
-				ImageId:      input.ImageId,
-				State: &ec2.InstanceState{
-					Code: aws.Int64(0),
-					Name: aws.String("pending"),
-				},
-			}
-		}
-
-		reservation := &ec2.Reservation{
-			ReservationId: aws.String("r-0123456789abcdef0"),
-			Instances:     instances,
-		}
-
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-
-	// Request 3 instances
-	input := createValidRunInstancesInput()
-	input.MinCount = aws.Int64(3)
-	input.MaxCount = aws.Int64(3)
-
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify multiple instances returned
-	require.NoError(t, err)
-	require.NotNil(t, reservation)
-	assert.Len(t, reservation.Instances, 3, "Should have 3 instances")
-}
-
-// TestNATSInstanceService_RunInstances_QueueGroup tests load balancing with queue groups.
-func TestNATSInstanceService_RunInstances_QueueGroup(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create multiple mock daemon workers in same queue group
-	worker1Count := 0
-	worker2Count := 0
-
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		worker1Count++
-		reservation := createValidReservation()
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		worker2Count++
-		reservation := createValidReservation()
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Send multiple requests
-	requestCount := 10
-	for range requestCount {
-		reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-		require.NoError(t, err)
-		require.NotNil(t, reservation)
-	}
-
-	// Verify load was distributed (both workers should have handled some requests)
-	// With queue groups, NATS distributes messages across subscribers
-	t.Logf("Worker 1 handled: %d requests", worker1Count)
-	t.Logf("Worker 2 handled: %d requests", worker2Count)
-
-	assert.Positive(t, worker1Count, "Worker 1 should handle some requests")
-	assert.Positive(t, worker2Count, "Worker 2 should handle some requests")
-	assert.Equal(t, requestCount, worker1Count+worker2Count, "All requests should be handled")
-}
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -25,7 +26,8 @@ import (
 // object store + memory meta) and its composed ECR JSON image-action service.
 func newImageGateway(t *testing.T) *ecrGW {
 	t.Helper()
-	meta := ecrdomain.NewMemoryMetaStore()
+	_, _, js := testutil.StartTestJetStream(t)
+	meta := ecrdomain.NewKVMetaStore(js)
 	reg := ecrregistry.NewRegistry(objectstore.NewMemoryObjectStore(), meta, ecrTestAccount)
 	gw := newECRGW(nil, awsapi.Deps{Registry: awsapi.NewRegistryActionService(reg, reg, reg, reg), LifecyclePreview: awsapi.NewLifecyclePreviewActionService(meta, reg)})
 	imageRegistries.Store(gw, reg)
@@ -5,8 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"maps"
-	"slices"
 	"sort"
 	"strings"
 	"sync"
@@ -593,295 +591,6 @@ func (s *KVMetaStore) DeleteUpload(ctx context.Context, accountID, uploadID stri
 	return nil
 }

-// MemoryMetaStore is an in-memory MetaStore used by tests and single-process
-// dev runs. It is safe for concurrent use.
-type MemoryMetaStore struct {
-	mu        sync.Mutex
-	repos     map[string]map[string]RepoMeta     // account -> repo -> meta
-	policies  map[string]map[string][]byte       // account -> repo -> policyText
-	lifecycle map[string]map[string][]byte       // account -> repo -> lifecyclePolicyText
-	tags      map[string]map[string]string       // account -> repo|tag -> digest
-	manifests map[string]map[string]ManifestMeta // account -> repo|digest -> meta
-	uploads   map[string]map[string]uploadRev    // account -> id -> state+rev
-}
-
-type uploadRev struct {
-	state UploadState
-	rev   uint64
-}
-
-var _ MetaStore = (*MemoryMetaStore)(nil)
-
-// NewMemoryMetaStore constructs an empty in-memory MetaStore.
-func NewMemoryMetaStore() *MemoryMetaStore {
-	return &MemoryMetaStore{
-		repos:     make(map[string]map[string]RepoMeta),
-		policies:  make(map[string]map[string][]byte),
-		lifecycle: make(map[string]map[string][]byte),
-		tags:      make(map[string]map[string]string),
-		manifests: make(map[string]map[string]ManifestMeta),
-		uploads:   make(map[string]map[string]uploadRev),
-	}
-}
-
-func (m *MemoryMetaStore) PutRepo(_ context.Context, accountID string, meta RepoMeta) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.repos[accountID] == nil {
-		m.repos[accountID] = make(map[string]RepoMeta)
-	}
-	m.repos[accountID][meta.Name] = meta
-	return nil
-}
-
-func (m *MemoryMetaStore) GetRepo(_ context.Context, accountID, repo string) (RepoMeta, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	r, ok := m.repos[accountID][repo]
-	if !ok {
-		return RepoMeta{}, ErrNotFound
-	}
-	return r, nil
-}
-
-func (m *MemoryMetaStore) ListRepos(_ context.Context, accountID string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return slices.Sorted(maps.Keys(m.repos[accountID])), nil
-}
-
-func (m *MemoryMetaStore) DeleteRepo(_ context.Context, accountID, repo string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if _, ok := m.repos[accountID][repo]; !ok {
-		return ErrNotFound
-	}
-	delete(m.repos[accountID], repo)
-	delete(m.policies[accountID], repo)
-	delete(m.lifecycle[accountID], repo)
-	for k := range m.tags[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo {
-			delete(m.tags[accountID], k)
-		}
-	}
-	for k := range m.manifests[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo {
-			delete(m.manifests[accountID], k)
-		}
-	}
-	return nil
-}
-
-func (m *MemoryMetaStore) ListManifests(_ context.Context, accountID, repo string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	var out []string
-	for k := range m.manifests[accountID] {
-		if r, d, ok := strings.Cut(k, "|"); ok && r == repo {
-			out = append(out, d)
-		}
-	}
-	sort.Strings(out)
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutRepoPolicy(_ context.Context, accountID, repo string, policyText []byte) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.policies[accountID] == nil {
-		m.policies[accountID] = make(map[string][]byte)
-	}
-	m.policies[accountID][repo] = policyText
-	return nil
-}
-
-func (m *MemoryMetaStore) GetRepoPolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.policies[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	return p, nil
-}
-
-func (m *MemoryMetaStore) DeleteRepoPolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.policies[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	delete(m.policies[accountID], repo)
-	return p, nil
-}
-
-func (m *MemoryMetaStore) PutLifecyclePolicy(_ context.Context, accountID, repo string, policyText []byte) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.lifecycle[accountID] == nil {
-		m.lifecycle[accountID] = make(map[string][]byte)
-	}
-	m.lifecycle[accountID][repo] = policyText
-	return nil
-}
-
-func (m *MemoryMetaStore) GetLifecyclePolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.lifecycle[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	return p, nil
-}
-
-func (m *MemoryMetaStore) DeleteLifecyclePolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.lifecycle[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	delete(m.lifecycle[accountID], repo)
-	return p, nil
-}
-
-func (m *MemoryMetaStore) PutTag(_ context.Context, accountID, repo, tag, digest string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.tags[accountID] == nil {
-		m.tags[accountID] = make(map[string]string)
-	}
-	m.tags[accountID][repo+"|"+tag] = digest
-	return nil
-}
-
-func (m *MemoryMetaStore) GetTag(_ context.Context, accountID, repo, tag string) (string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	d, ok := m.tags[accountID][repo+"|"+tag]
-	if !ok {
-		return "", ErrNotFound
-	}
-	return d, nil
-}
-
-func (m *MemoryMetaStore) DeleteTag(_ context.Context, accountID, repo, tag string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	key := repo + "|" + tag
-	if _, ok := m.tags[accountID][key]; !ok {
-		return ErrNotFound
-	}
-	delete(m.tags[accountID], key)
-	return nil
-}
-
-func (m *MemoryMetaStore) ListTags(_ context.Context, accountID, repo string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	var out []string
-	for k := range m.tags[accountID] {
-		if r, t, ok := strings.Cut(k, "|"); ok && r == repo {
-			out = append(out, t)
-		}
-	}
-	sort.Strings(out)
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutManifestMeta(_ context.Context, accountID, repo string, meta ManifestMeta) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.manifests[accountID] == nil {
-		m.manifests[accountID] = make(map[string]ManifestMeta)
-	}
-	m.manifests[accountID][repo+"|"+meta.Digest] = meta
-	return nil
-}
-
-func (m *MemoryMetaStore) GetManifestMeta(_ context.Context, accountID, repo, digest string) (ManifestMeta, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	meta, ok := m.manifests[accountID][repo+"|"+digest]
-	if !ok {
-		return ManifestMeta{}, ErrNotFound
-	}
-	return meta, nil
-}
-
-func (m *MemoryMetaStore) DeleteManifestMeta(_ context.Context, accountID, repo, digest string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	key := repo + "|" + digest
-	if _, ok := m.manifests[accountID][key]; !ok {
-		return ErrNotFound
-	}
-	delete(m.manifests[accountID], key)
-	return nil
-}
-
-func (m *MemoryMetaStore) ListReferrers(_ context.Context, accountID, repo, subjectDigest string) ([]ReferrerDescriptor, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	out := []ReferrerDescriptor{}
-	for k, meta := range m.manifests[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo && meta.SubjectDigest == subjectDigest {
-			out = append(out, meta.referrerDescriptor())
-		}
-	}
-	sort.Slice(out, func(i, j int) bool { return out[i].Digest < out[j].Digest })
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutUpload(_ context.Context, accountID, uploadID string, state UploadState) (uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.uploads[accountID] == nil {
-		m.uploads[accountID] = make(map[string]uploadRev)
-	}
-	rev := m.uploads[accountID][uploadID].rev + 1
-	m.uploads[accountID][uploadID] = uploadRev{state: state, rev: rev}
-	return rev, nil
-}
-
-func (m *MemoryMetaStore) GetUpload(_ context.Context, accountID, uploadID string) (UploadState, uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	u, ok := m.uploads[accountID][uploadID]
-	if !ok {
-		return UploadState{}, 0, ErrNotFound
-	}
-	return u.state, u.rev, nil
-}
-
-func (m *MemoryMetaStore) UpdateUpload(_ context.Context, accountID, uploadID string, state UploadState, rev uint64) (uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	u, ok := m.uploads[accountID][uploadID]
-	if !ok {
-		return 0, ErrNotFound
-	}
-	if u.rev != rev {
-		return 0, ErrConflict
-	}
-	u.state = state
-	u.rev = rev + 1
-	m.uploads[accountID][uploadID] = u
-	return u.rev, nil
-}
-
-func (m *MemoryMetaStore) DeleteUpload(_ context.Context, accountID, uploadID string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if _, ok := m.uploads[accountID][uploadID]; !ok {
-		return ErrNotFound
-	}
-	delete(m.uploads[accountID], uploadID)
-	return nil
-}
-
 // mapKVErr normalizes JetStream KV errors to the MetaStore error vocabulary.
 func mapKVErr(err error) error {
 	if errors.Is(err, jetstream.ErrKeyNotFound) {
@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"

+	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -61,8 +62,8 @@ func TestKVKeyHelpers(t *testing.T) {
 	assert.Equal(t, "sha256-abc", DigestToken("sha256:abc"))
 }

-func TestMemoryMetaStore_RepoRoundtrip(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_RepoRoundtrip(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"

 	_, err := m.GetRepo(context.Background(), acct, "team/app")
@@ -80,8 +81,8 @@ func TestMemoryMetaStore_RepoRoundtrip(t *testing.T) {
 	assert.Equal(t, []string{"team/app", "team/web"}, repos)
 }

-func TestMemoryMetaStore_TagsAndManifests(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_TagsAndManifests(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"

 	require.NoError(t, m.PutTag(context.Background(), acct, "r", "v1", "sha256:aaa"))
@@ -112,8 +113,8 @@ func TestMemoryMetaStore_TagsAndManifests(t *testing.T) {
 	assert.ErrorIs(t, m.DeleteManifestMeta(context.Background(), acct, "r", "sha256:ccc"), ErrNotFound)
 }

-func TestMemoryMetaStore_UploadCAS(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_UploadCAS(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"

 	rev, err := m.PutUpload(context.Background(), acct, "u1", UploadState{RepoName: "r"})
@@ -143,8 +144,8 @@ func TestTrimSuffixMeta(t *testing.T) {
 	assert.Equal(t, "team/app", trimSuffixMeta("repos/team/app/meta"))
 }

-func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_DeleteRepoAndListManifests(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"

 	require.NoError(t, m.PutRepo(context.Background(), acct, RepoMeta{Name: "team/app", CreatedAt: time.Now()}))
@@ -156,7 +157,7 @@ func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {

 	digs, err := m.ListManifests(context.Background(), acct, "team/app")
 	require.NoError(t, err)
-	assert.Equal(t, []string{"sha256:ccc"}, digs)
+	assert.Equal(t, []string{DigestToken("sha256:ccc")}, digs)

 	require.NoError(t, m.DeleteRepo(context.Background(), acct, "team/app"))
 	_, err = m.GetRepo(context.Background(), acct, "team/app")
@@ -180,50 +181,6 @@ func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
 	assert.ErrorIs(t, m.DeleteRepo(context.Background(), acct, "team/ghost"), ErrNotFound)
 }

-func TestMemoryMetaStore_RepoPolicy(t *testing.T) {
-	m := NewMemoryMetaStore()
-	const acct = "000000000000"
-	const policy = `{"Version":"2012-10-17"}`
-
-	_, err := m.GetRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-	_, err = m.DeleteRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-
-	require.NoError(t, m.PutRepoPolicy(context.Background(), acct, "team/app", []byte(policy)))
-	got, err := m.GetRepoPolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(got))
-
-	deleted, err := m.DeleteRepoPolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(deleted))
-	_, err = m.GetRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-}
-
-func TestMemoryMetaStore_LifecyclePolicy(t *testing.T) {
-	m := NewMemoryMetaStore()
-	const acct = "000000000000"
-	const policy = `{"rules":[]}`
-
-	_, err := m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-	_, err = m.DeleteLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-
-	require.NoError(t, m.PutLifecyclePolicy(context.Background(), acct, "team/app", []byte(policy)))
-	got, err := m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(got))
-
-	deleted, err := m.DeleteLifecyclePolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(deleted))
-	_, err = m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-}
-
 func TestManifestMeta_DecodesRecordWithoutReferrerFields(t *testing.T) {
 	var m ManifestMeta
 	require.NoError(t, json.Unmarshal([]byte(`{"digest":"sha256:ccc","mediaType":"application/vnd.oci.image.manifest.v1+json","size":10,"pushedAt":"2026-01-01T00:00:00Z","childDigests":["sha256:ddd"]}`), &m))
@@ -232,3 +189,9 @@ func TestManifestMeta_DecodesRecordWithoutReferrerFields(t *testing.T) {
 	assert.Empty(t, m.SubjectDigest)
 	assert.Empty(t, m.ArtifactType)
 }
+
+func newTestKVMetaStore(t *testing.T) *KVMetaStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	return NewKVMetaStore(js)
+}
@@ -4,7 +4,6 @@ import (
 	"context"
 	"testing"

-	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -30,11 +29,7 @@ func referrerMeta(digest, subject string) ManifestMeta {
 // its own referrers state.
 func referrerStores(t *testing.T) map[string]MetaStore {
 	t.Helper()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	return map[string]MetaStore{
-		"memory": NewMemoryMetaStore(),
-		"kv":     NewKVMetaStore(testutil.NewJetStream(t, nc)),
-	}
+	return map[string]MetaStore{"kv": newTestKVMetaStore(t)}
 }

 func TestListReferrers(t *testing.T) {
@@ -50,7 +50,7 @@ func putPolicy(t *testing.T, reg *Registry, account, repo, policy string) {
 const untaggedExpirePolicy = `{"rules":[{"rulePriority":1,"selection":{"tagStatus":"untagged","countType":"sinceImagePushed","countUnit":"days","countNumber":7},"action":{"type":"expire"}}]}`

 func TestSweepRepo_SinceImagePushed(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -65,7 +65,7 @@ func TestSweepRepo_SinceImagePushed(t *testing.T) {
 }

 func TestSweepRepo_ImageCountMoreThan(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:a", now.AddDate(0, 0, -1), "a")
@@ -81,7 +81,7 @@ func TestSweepRepo_ImageCountMoreThan(t *testing.T) {
 }

 func TestSweepRepo_NoPolicy(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:x", now.AddDate(0, 0, -100))
@@ -92,7 +92,7 @@ func TestSweepRepo_NoPolicy(t *testing.T) {
 }

 func TestSweepRepo_Idempotent(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -105,7 +105,7 @@ func TestSweepRepo_Idempotent(t *testing.T) {
 }

 func TestSweepOnce_MultiAccount(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	const acct2 = "000000000001"
 	policy := `{"rules":[{"rulePriority":1,"selection":{"tagStatus":"any","countType":"imageCountMoreThan","countNumber":1},"action":{"type":"expire"}}]}`
@@ -125,7 +125,7 @@ func TestSweepOnce_MultiAccount(t *testing.T) {
 }

 func TestSweepOnce_AccountsError(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	s := NewLifecycleSweeper(reg, func() ([]string, error) {
 		return nil, assert.AnError
 	}, time.Hour)
@@ -133,7 +133,7 @@ func TestSweepOnce_AccountsError(t *testing.T) {
 }

 func TestNewLifecycleSweeper_DefaultInterval(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	s := NewLifecycleSweeper(reg, func() ([]string, error) { return nil, nil }, 0)
 	assert.Equal(t, DefaultLifecycleSweepInterval, s.interval)

@@ -147,7 +147,7 @@ func TestNewLifecycleSweeper_DefaultInterval(t *testing.T) {
 // TestSweeperRun drives the ticker loop: with a tiny interval and an expiring
 // image, the first tick must delete it; cancelling the context stops Run.
 func TestSweeperRun(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -184,7 +184,7 @@ func TestSweeperRun(t *testing.T) {
 // TestSweepRepo_InvalidStoredPolicy exercises the evaluate-error branch: a
 // malformed stored policy is logged and skipped, deleting nothing.
 func TestSweepRepo_InvalidStoredPolicy(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:x", now.AddDate(0, 0, -100))
@@ -198,7 +198,7 @@ func TestSweepRepo_InvalidStoredPolicy(t *testing.T) {
 // TestSweepRepo_PolicyWithoutRepo exercises the ListImages-not-found branch: a
 // policy stored for a repo with no manifests yields nothing to expire.
 func TestSweepRepo_PolicyWithoutRepo(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	putPolicy(t, reg, testAccount, "ghost", untaggedExpirePolicy)

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"testing"

-	"github.com/mulgadc/spinifex/spinifex/domains/ecr"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -32,7 +31,7 @@ func (c *countingStore) EnsureBucket(ctx context.Context, bucket string) error {

 func TestRegistry_EnsureBucketFailureReturns500(t *testing.T) {
 	store := &countingStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), ensureErr: errors.New("predastore down")}
-	reg := NewRegistry(store, ecr.NewMemoryMetaStore(), testAccount)
+	reg := NewRegistry(store, newTestMetaStore(t), testAccount)

 	w := do(reg, http.MethodGet, "/v2/myrepo/tags/list", nil, nil)
 	assert.Equal(t, http.StatusInternalServerError, w.Code)
@@ -40,7 +39,7 @@ func TestRegistry_EnsureBucketFailureReturns500(t *testing.T) {

 func TestRegistry_EnsureBucketCachedAfterSuccess(t *testing.T) {
 	store := &countingStore{MemoryObjectStore: objectstore.NewMemoryObjectStore()}
-	reg := NewRegistry(store, ecr.NewMemoryMetaStore(), testAccount)
+	reg := NewRegistry(store, newTestMetaStore(t), testAccount)

 	do(reg, http.MethodGet, "/v2/myrepo/tags/list", nil, nil)
 	do(reg, http.MethodGet, "/v2/other/tags/list", nil, nil)
@@ -44,7 +44,7 @@ func storeLayersManifest(t *testing.T, reg *Registry, repo, tag string, layers .
 }

 func TestDeleteImage_ReclaimsBlobsKeepsShared(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	shared := pushBlob(t, reg, repo, []byte("shared-layer-bytes"))
 	exclusive := pushBlob(t, reg, repo, []byte("exclusive-layer-bytes"))
@@ -85,7 +85,7 @@ func seedImage(t *testing.T, reg *Registry, repo, tag string) ([]byte, string) {
 }

 func TestStoreAndGetManifest_RoundTrip(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	manifest, digest := seedImage(t, reg, "team/app", "v1")

 	// Fetch by tag.
@@ -114,7 +114,7 @@ func TestStoreAndGetManifest_RoundTrip(t *testing.T) {
 }

 func TestListImages_TaggedAndUntagged(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	seedImage(t, reg, repo, "v1")
 	seedImage(t, reg, repo, "v2")
@@ -176,7 +176,7 @@ func TestListImages_KVDigestForm(t *testing.T) {
 }

 func TestDeleteImage_ByTagAndByDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	_, digest := seedImage(t, reg, repo, "v1")
 	seedImage(t, reg, repo, "v2") // shares nothing; distinct digest
@@ -205,7 +205,7 @@ func TestDeleteImage_ByTagAndByDigest(t *testing.T) {
 }

 func TestDeleteImage_ByDigestClearsTags(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	_, digest := seedImage(t, reg, repo, "v1")
 	// Add a second tag pointing at the same digest.
@@ -221,7 +221,7 @@ func TestDeleteImage_ByDigestClearsTags(t *testing.T) {
 }

 func TestStoreManifest_ImmutableTag(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	manA := fmt.Appendf(nil, `{"schemaVersion":2,"mediaType":"%s","layers":[{"digest":"%s"}]}`,
 		mediaTypeDockerManifest, pushBlob(t, reg, repo, []byte("layer-a")))
@@ -254,7 +254,7 @@ func TestStoreManifest_ImmutableTag(t *testing.T) {
 }

 func TestStoreManifest_BadDigestReference(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfgDg := pushBlob(t, reg, repo, []byte("cfg"))
 	manifest := fmt.Appendf(nil,
@@ -38,7 +38,7 @@ func storedMeta(t *testing.T, reg *Registry, body string) (subject, artifactType
 }

 func TestPutManifest_SubjectRecordedAndAdvertised(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))

 	tests := []struct {
@@ -65,7 +65,7 @@ func TestPutManifest_SubjectRecordedAndAdvertised(t *testing.T) {
 }

 func TestPutManifest_IndexSubject(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)

 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"manifests":[],"subject":{"mediaType":%q,"digest":%q,"size":10}}`,
@@ -81,7 +81,7 @@ func TestPutManifest_IndexSubject(t *testing.T) {
 }

 func TestPutManifest_NoSubjectLeavesFieldsEmpty(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"artifactType":%q,"config":{"mediaType":%q,"digest":%q,"size":3},"layers":[]}`,
 		mediaTypeOCIManifest, sigArtifactType, sigConfigType, cfg)
@@ -97,7 +97,7 @@ func TestPutManifest_NoSubjectLeavesFieldsEmpty(t *testing.T) {
 }

 func TestPutManifest_MalformedSubjectDigestRejected(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := imageWithSubject(cfg, "sha256:nothex", "")

@@ -111,7 +111,7 @@ func TestPutManifest_MalformedSubjectDigestRejected(t *testing.T) {
 }

 func TestPutManifest_IndexWithContentTypeParameterGetsNoConfigArtifactType(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)

 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"manifests":[],"subject":{"mediaType":%q,"digest":%q,"size":10}}`,
@@ -128,7 +128,7 @@ func TestPutManifest_IndexWithContentTypeParameterGetsNoConfigArtifactType(t *te
 }

 func TestDeleteReferrerManifest_RemovesItFromTheSubjectListing(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := imageWithSubject(cfg, absentSubjectDgt, sigArtifactType)
 	dg := digestOf([]byte(body))
@@ -183,7 +183,7 @@ func getReferrersBody(t *testing.T, reg *Registry, path string) (*httptest.Respo
 }

 func TestGetReferrers_ListsReferrersAsAnOCIIndex(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dgA := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	dgB := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, "application/vnd.example.sbom.v1", "cfgB")

@@ -202,7 +202,7 @@ func TestGetReferrers_ListsReferrersAsAnOCIIndex(t *testing.T) {
 }

 func TestGetReferrers_NoReferrersIsAnEmptyIndexNotAnError(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)

 	w := do(reg, http.MethodGet, "/v2/"+referrerRepo+"/referrers/"+absentSubjectDgt, nil, nil)
@@ -212,7 +212,7 @@ func TestGetReferrers_NoReferrersIsAnEmptyIndexNotAnError(t *testing.T) {
 }

 func TestGetReferrers_ArtifactTypeFilter(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dgSig := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	pushReferrer(t, reg, referrerRepo, absentSubjectDgt, "application/vnd.example.sbom.v1", "cfgB")

@@ -230,7 +230,7 @@ func TestGetReferrers_ArtifactTypeFilter(t *testing.T) {
 }

 func TestGetReferrers_MalformedDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)

 	w := do(reg, http.MethodGet, "/v2/"+referrerRepo+"/referrers/not-a-digest", nil, nil)
@@ -239,14 +239,14 @@ func TestGetReferrers_MalformedDigest(t *testing.T) {
 }

 func TestGetReferrers_UnknownRepository(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/ghost/referrers/"+absentSubjectDgt, nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "NAME_UNKNOWN")
 }

 func TestGetReferrers_DoesNotCrossRepositories(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	seedRepo(t, reg, "team/other")

@@ -256,7 +256,7 @@ func TestGetReferrers_DoesNotCrossRepositories(t *testing.T) {
 }

 func TestReferrersRoute_OnlyGET(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 	for _, method := range []string{http.MethodPut, http.MethodDelete, http.MethodPost, http.MethodPatch} {
 		w := do(reg, method, "/v2/"+referrerRepo+"/referrers/"+absentSubjectDgt, []byte("{}"), nil)
@@ -272,7 +272,7 @@ func (failingReferrerStore) ListReferrers(context.Context, string, string, strin
 }

 func TestGetReferrers_MetadataFailureIsA5xxNotA404(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 	reg.Meta = failingReferrerStore{reg.Meta}

@@ -310,7 +310,7 @@ func referrerDigests(t *testing.T, reg *Registry, repo, subject string) []string
 }

 func TestDeleteManifest_SubjectDeleteRemovesItsReferrers(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	sbom := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "sbom")
@@ -326,7 +326,7 @@ func TestDeleteManifest_SubjectDeleteRemovesItsReferrers(t *testing.T) {
 }

 func TestDeleteManifest_NestedReferrersAreRemoved(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	sigOfSig := pushReferrer(t, reg, referrerRepo, sig, sigArtifactType, "sig-of-sig")
@@ -341,7 +341,7 @@ func TestDeleteManifest_NestedReferrersAreRemoved(t *testing.T) {
 }

 func TestDeleteManifest_ReferrerDeleteLeavesTheSubject(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	other := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "sbom")
@@ -355,7 +355,7 @@ func TestDeleteManifest_ReferrerDeleteLeavesTheSubject(t *testing.T) {
 }

 func TestDeleteManifest_TagDeleteDoesNotCascade(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	require.NoError(t, reg.Meta.PutTag(context.Background(), testAccount, referrerRepo, "v1", subject))
@@ -368,7 +368,7 @@ func TestDeleteManifest_TagDeleteDoesNotCascade(t *testing.T) {
 }

 func TestDeleteManifest_CascadeStaysInsideTheRepository(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	elsewhere := pushReferrer(t, reg, "team/other", subject, sigArtifactType, "sig-other")
@@ -394,7 +394,7 @@ func (s failingDeleteStore) DeleteManifestMeta(ctx context.Context, account, rep
 }

 func TestDeleteManifest_ReferrerFailureDoesNotFailTheSubjectDelete(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	stuck := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "stuck")
 	fine := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "fine")
@@ -2,6 +2,7 @@ package ecrregistry

 import (
 	"context"
+	"github.com/mulgadc/spinifex/internal/testkit"

 	"crypto/sha256"
 	"encoding/hex"
@@ -21,8 +22,15 @@ import (

 const testAccount = "000000000000"

-func newTestRegistry() *Registry {
-	return NewRegistry(objectstore.NewMemoryObjectStore(), ecr.NewMemoryMetaStore(), testAccount)
+func newTestRegistry(t *testing.T) *Registry {
+	t.Helper()
+	return NewRegistry(objectstore.NewMemoryObjectStore(), newTestMetaStore(t), testAccount)
+}
+
+func newTestMetaStore(t *testing.T) *ecr.KVMetaStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	return ecr.NewKVMetaStore(js)
 }

 func digestOf(b []byte) string {
@@ -69,7 +77,7 @@ func pushBlob(t *testing.T, reg *Registry, repo string, data []byte) string {
 }

 func TestBlobUpload_Monolithic_And_Head_Get(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("hello layer bytes")
 	dg := pushBlob(t, reg, "team/app", data)

@@ -84,20 +92,20 @@ func TestBlobUpload_Monolithic_And_Head_Get(t *testing.T) {
 }

 func TestBlobHead_NotFound(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodHead, "/v2/team/app/blobs/"+digestOf([]byte("absent")), nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }

 func TestBlobGet_MalformedDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/blobs/sha256:zzz", nil, nil)
 	assert.Equal(t, http.StatusBadRequest, w.Code)
 	assertCode(t, w, "DIGEST_INVALID")
 }

 func TestBlobUpload_Chunked(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	c1 := []byte("first-chunk-")
 	c2 := []byte("second-chunk")
@@ -124,7 +132,7 @@ func TestBlobUpload_Chunked(t *testing.T) {
 }

 func TestBlobUpload_DigestMismatch(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	w := do(reg, http.MethodPost, "/v2/r/repo/blobs/uploads/", nil, nil)
 	loc := w.Header().Get("Location")
@@ -134,7 +142,7 @@ func TestBlobUpload_DigestMismatch(t *testing.T) {
 }

 func TestBlobUpload_Cancel(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	w := do(reg, http.MethodPost, "/v2/r/repo/blobs/uploads/", nil, nil)
 	loc := w.Header().Get("Location")
@@ -161,7 +169,7 @@ func putManifestReferencing(t *testing.T, reg *Registry, repo, digest string) {
 }

 func TestBlobMount_HitAndMiss(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("shared-layer")
 	dg := pushBlob(t, reg, "team/source", data)
 	putManifestReferencing(t, reg, "team/source", dg)
@@ -185,7 +193,7 @@ func TestBlobMount_HitAndMiss(t *testing.T) {
 // as a plain upload start, matching the authorization middleware's
 // classification of the same request.
 func TestBlobMount_MissingFromFallsThroughToUpload(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("shared-layer")
 	dg := pushBlob(t, reg, "team/source", data)
 	putManifestReferencing(t, reg, "team/source", dg)
@@ -200,7 +208,7 @@ func TestBlobMount_MissingFromFallsThroughToUpload(t *testing.T) {
 // named source repository is rejected as a mount — proving digest existence
 // alone is not enough to mount cross-repo.
 func TestBlobMount_UnreferencedDigestFallsThrough(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	unreferenced := pushBlob(t, reg, "team/other", []byte("not-in-source"))
 	seedRepo(t, reg, "team/source")
 	seedRepo(t, reg, "team/dest")
@@ -214,7 +222,7 @@ func TestBlobMount_UnreferencedDigestFallsThrough(t *testing.T) {
 // repository that does not exist never mounts, and never distinguishes
 // itself from an unreferenced-digest miss (both fall through identically).
 func TestBlobMount_NonexistentSourceFallsThrough(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dg := pushBlob(t, reg, "team/elsewhere", []byte("some-bytes"))
 	seedRepo(t, reg, "team/dest")

@@ -224,7 +232,7 @@ func TestBlobMount_NonexistentSourceFallsThrough(t *testing.T) {
 }

 func TestManifest_PutGetHead_ImageManifest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"

 	cfg := []byte(`{"config":true}`)
@@ -255,7 +263,7 @@ func TestManifest_PutGetHead_ImageManifest(t *testing.T) {
 }

 func TestManifest_Put_MissingBlob(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	seedRepo(t, reg, repo)
 	manifest := fmt.Appendf(nil,
@@ -268,7 +276,7 @@ func TestManifest_Put_MissingBlob(t *testing.T) {
 }

 func TestManifest_Get_AcceptMismatch(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -285,7 +293,7 @@ func TestManifest_Get_AcceptMismatch(t *testing.T) {
 // acceptable manifest media type as its own Accept header line rather than one
 // comma-joined value. Negotiation must consider every line, not just the first.
 func TestManifest_Get_MultipleAcceptHeaders(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -305,7 +313,7 @@ func TestManifest_Get_MultipleAcceptHeaders(t *testing.T) {
 }

 func TestManifest_Get_Unknown(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/manifests/missing", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "MANIFEST_UNKNOWN")
@@ -314,7 +322,7 @@ func TestManifest_Get_Unknown(t *testing.T) {
 // TestStartUpload_RepoNotCreated asserts a blob upload to a repository that was
 // never created is rejected up front — ECR does not auto-create on push.
 func TestStartUpload_RepoNotCreated(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodPost, "/v2/team/ghost/blobs/uploads/", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "NAME_UNKNOWN")
@@ -323,7 +331,7 @@ func TestStartUpload_RepoNotCreated(t *testing.T) {
 // TestPutManifest_RepoNotCreated asserts a manifest PUT to an uncreated
 // repository is rejected with NAME_UNKNOWN.
 func TestPutManifest_RepoNotCreated(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	manifest := fmt.Appendf(nil, `{"mediaType":"%s","layers":[]}`, mediaTypeDockerManifest)
 	w := do(reg, http.MethodPut, "/v2/team/ghost/manifests/v1", manifest,
 		map[string]string{"Content-Type": mediaTypeDockerManifest})
@@ -332,7 +340,7 @@ func TestPutManifest_RepoNotCreated(t *testing.T) {
 }

 func TestManifest_ImageIndex_ChildValidation(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"

 	// Push a child image manifest first.
@@ -372,7 +380,7 @@ func TestManifest_ImageIndex_ChildValidation(t *testing.T) {
 }

 func TestManifest_Delete_Tag(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -390,7 +398,7 @@ func TestManifest_Delete_Tag(t *testing.T) {
 // TestManifest_Delete_ByDigest proves delete-by-digest is no longer a no-op: it
 // removes the manifest record + object, and a subsequent GET is a 404.
 func TestManifest_Delete_ByDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("cfg-bytes"))
 	manifest := fmt.Appendf(nil,
@@ -412,7 +420,7 @@ func TestManifest_Delete_ByDigest(t *testing.T) {
 }

 func TestTagsList(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -434,13 +442,13 @@ func TestTagsList(t *testing.T) {
 }

 func TestTagsList_UnknownRepo(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/ghost/tags/list", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }

 func TestCatalog(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	pushBlob(t, reg, "team/a", []byte("a"))
 	pushBlob(t, reg, "team/b", []byte("b"))

@@ -454,21 +462,21 @@ func TestCatalog(t *testing.T) {
 }

 func TestDispatch_InvalidRepoName(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/Team/App/tags/list", nil, nil)
 	assert.Equal(t, http.StatusBadRequest, w.Code)
 	assertCode(t, w, "NAME_INVALID")
 }

 func TestDispatch_RepeatedSeparatorRepoName(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dg := pushBlob(t, reg, "team/a__b--c", []byte("layer"))
 	w := do(reg, http.MethodHead, "/v2/team/a__b--c/blobs/"+dg, nil, nil)
 	assert.Equal(t, http.StatusOK, w.Code)
 }

 func TestDispatch_UnknownPath(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/frobnicate", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }
@@ -35,7 +35,7 @@ func (s *blobPutFailStore) PutObject(ctx context.Context, input *s3.PutObjectInp
 func TestFinishUpload_StoreFailure_CleansUp(t *testing.T) {
 	mem := objectstore.NewMemoryObjectStore()
 	store := &blobPutFailStore{MemoryObjectStore: mem}
-	meta := ecr.NewMemoryMetaStore()
+	meta := newTestMetaStore(t)
 	reg := NewRegistry(store, meta, testAccount)
 	seedRepo(t, reg, "team/app")

@@ -72,7 +72,7 @@ func awsString(s string) *string { return &s }
 // updates issued from the same observed revision cannot both commit. One wins,
 // the other gets ErrConflict — so a racing PATCH is rejected, never merged.
 func TestUploadCAS_SameRevisionSingleWinner(t *testing.T) {
-	meta := ecr.NewMemoryMetaStore()
+	meta := newTestMetaStore(t)
 	rev, err := meta.PutUpload(context.Background(), testAccount, "u-race", ecr.UploadState{RepoName: "r"})
 	require.NoError(t, err)

@@ -116,7 +116,7 @@ func TestUploadCAS_SameRevisionSingleWinner(t *testing.T) {
 // hash state and the recorded byte object never desynchronise.
 func TestPatchUpload_ConcurrentNoCorruption(t *testing.T) {
 	for iter := range 25 {
-		reg := newTestRegistry()
+		reg := newTestRegistry(t)
 		repo := "team/race"
 		seedRepo(t, reg, repo)

@@ -16,8 +16,7 @@ type MetaServiceImpl struct {

 var _ MetaService = (*MetaServiceImpl)(nil)

-// NewMetaServiceImpl builds a MetaService over an explicit MetaStore (used by
-// daemon-side tests with a MemoryMetaStore).
+// NewMetaServiceImpl builds a MetaService over an explicit MetaStore.
 func NewMetaServiceImpl(store MetaStore) *MetaServiceImpl {
 	return &MetaServiceImpl{store: store}
 }
@@ -21,7 +21,7 @@ const svcTestAccount = "000000000000"
 // ErrNotFound/ErrConflict into the response flags rather than transport errors,
 // so they survive the AWS-error-code-only NATS reply.
 func TestMetaServiceImpl_FlagMapping(t *testing.T) {
-	svc := NewMetaServiceImpl(NewMemoryMetaStore())
+	svc := NewMetaServiceImpl(newTestKVMetaStore(t))

 	// Absent repo -> Found:false, nil error.
 	dr, err := svc.RepoDescribe(context.Background(), &RepoDescribeRequest{Repo: "ghost"}, svcTestAccount)
@@ -205,6 +205,8 @@ func TestS2_EKSLastNodegroupDeletedResumesTheClusterDelete(t *testing.T) {
 		ClusterName: aws.String("alpha"), NodegroupName: aws.String("ng1"),
 	}, testAccountID)
 	require.NoError(t, err)
+	_, err = f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
+	require.NoError(t, err)
 	f.svc.WaitTeardowns()

 	_, err = GetClusterMeta(t.Context(), f.kv, "alpha")
@@ -105,6 +105,9 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture {
 	require.NoError(t, err)
 	// Tight Ready-gating knobs so nodegroup launches resolve in tests without the
 	// production 10m timeout; tests drive meta.NodeCount to simulate the reconciler.
+	svc.noDeleteKick = true
+	svc.workersEndWait = 20 * time.Millisecond
+	svc.workersEndPoll = time.Millisecond
 	svc.nodegroupReadyTimeout = 50 * time.Millisecond
 	svc.nodegroupReadyPoll = 2 * time.Millisecond
 	// Tight worker-launch-retry knobs so a launch-failure retry test resolves
@@ -0,0 +1,284 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// A DELETING nodegroup becomes DELETE_FAILED once it has been deleting this
+// long and this many passes have failed. Vars for tests.
+var (
+	nodegroupDeleteFailAfter    = 30 * time.Minute
+	nodegroupDeleteFailAttempts = 5
+)
+
+// A pass waits this long, polling, for ended workers to be proven gone.
+const (
+	defaultWorkersEndWait = 90 * time.Second
+	defaultWorkersEndPoll = 3 * time.Second
+)
+
+// errWorkersEnding means workers were told to end but are not yet proven gone.
+var errWorkersEnding = errors.New("workers still ending")
+
+// EKSNodegroupDeletingReaper finishes recorded nodegroup deletes: it ends the
+// workers, removes the nodegroup's own instance profile and only then the
+// record, so a nodegroup is never gone while something it made still is.
+type EKSNodegroupDeletingReaper struct {
+	svc *EKSServiceImpl
+}
+
+var _ vm.Reaper = (*EKSNodegroupDeletingReaper)(nil)
+
+// NewNodegroupDeletingReaper builds the DELETING nodegroup converger.
+func (s *EKSServiceImpl) NewNodegroupDeletingReaper() *EKSNodegroupDeletingReaper {
+	return &EKSNodegroupDeletingReaper{svc: s}
+}
+
+func (r *EKSNodegroupDeletingReaper) Class() string         { return "eks-nodegroup-deleting" }
+func (r *EKSNodegroupDeletingReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
+
+// Sweep hands every due DELETING nodegroup to a background converger and
+// returns how many it started. The lease decides which node runs each one.
+func (r *EKSNodegroupDeletingReaper) Sweep(ctx context.Context) (int, error) {
+	if !r.svc.depsReadyForOrchestration() {
+		return 0, nil
+	}
+	js, err := jetstream.New(r.svc.deps.NATSConn)
+	if err != nil {
+		return 0, fmt.Errorf("jetstream: %w", err)
+	}
+	names, err := accountBucketNames(ctx, r.svc.deps.NATSConn)
+	if err != nil {
+		return 0, fmt.Errorf("enumerate account buckets: %w", err)
+	}
+
+	started := 0
+	for _, name := range names {
+		select {
+		case <-ctx.Done():
+			return started, ctx.Err()
+		default:
+		}
+		accountID := strings.TrimPrefix(name, KVBucketEKSAccountPrefix)
+		acctKV, err := js.KeyValue(ctx, name)
+		if err != nil {
+			continue
+		}
+		clusters, err := listClusterNames(ctx, acctKV)
+		if err != nil {
+			continue
+		}
+		for _, cluster := range clusters {
+			recs, err := ListNodegroupRecords(ctx, acctKV, cluster)
+			if err != nil {
+				continue
+			}
+			for _, rec := range recs {
+				if r.due(rec) && r.svc.startNodegroupDelete(accountID, acctKV, cluster, rec.Name) {
+					started++
+				}
+			}
+		}
+	}
+	return started, nil
+}
+
+// due reports whether rec is DELETING and past its retry backoff.
+func (r *EKSNodegroupDeletingReaper) due(rec *NodegroupRecord) bool {
+	if rec.Status != eks.NodegroupStatusDeleting {
+		return false
+	}
+	if rec.DeleteReapAttempts == 0 || rec.LastDeleteReapAttempt.IsZero() {
+		return true
+	}
+	return time.Since(rec.LastDeleteReapAttempt) >= deleteReapBackoff(deletingReapMinAge, rec.DeleteReapAttempts)
+}
+
+// startNodegroupDelete runs one converger pass in the background and reports
+// whether it started; a pass already running here or on another node refuses.
+func (s *EKSServiceImpl) startNodegroupDelete(accountID string, kv jetstream.KeyValue, cluster, ng string) bool {
+	release, _, ok := s.holdTeardown(accountID, cluster+"/nodegroup/"+ng, false)
+	if !ok {
+		return false
+	}
+	s.teardownWG.Go(func() {
+		defer release()
+		ctx, cancel := context.WithTimeout(s.bgCtx, teardownPassTimeout)
+		defer cancel()
+		if err := s.convergeNodegroupDeletion(ctx, accountID, kv, cluster, ng); err != nil {
+			slog.Warn("eks: nodegroup delete incomplete, the record stays DELETING for retry",
+				"cluster", cluster, "nodegroup", ng, "account", accountID, "err", err)
+		}
+	})
+	return true
+}
+
+// WaitNodegroupDeletes blocks until every converger this process started has
+// returned. Intended for tests.
+func (s *EKSServiceImpl) WaitNodegroupDeletes() { s.teardownWG.Wait() }
+
+// mutateNodegroup applies fn to the stored record under compare-and-swap. fn
+// returns false to leave the record unchanged.
+func (s *EKSServiceImpl) mutateNodegroup(ctx context.Context, kv jetstream.KeyValue, cluster, ng string, fn func(*NodegroupRecord) bool) (*NodegroupRecord, error) {
+	for range ngCASMaxRetries {
+		rec, rev, err := getNodegroupEntry(ctx, kv, cluster, ng)
+		if err != nil {
+			return nil, err
+		}
+		if !fn(rec) {
+			return rec, nil
+		}
+		ok, err := s.casPutNodegroup(ctx, kv, rec, rev)
+		if err != nil {
+			return nil, err
+		}
+		if ok {
+			return rec, nil
+		}
+	}
+	return nil, fmt.Errorf("nodegroup %s/%s: concurrent writers kept winning", cluster, ng)
+}
+
+// convergeNodegroupDeletion finishes one recorded delete. It acts only on a
+// record that is DELETING when re-read, and keeps the record on any failure.
+func (s *EKSServiceImpl) convergeNodegroupDeletion(ctx context.Context, accountID string, kv jetstream.KeyValue, cluster, ng string) error {
+	rec, err := s.mutateNodegroup(ctx, kv, cluster, ng, func(r *NodegroupRecord) bool {
+		if r.Status != eks.NodegroupStatusDeleting {
+			return false
+		}
+		r.DeleteReapAttempts++
+		r.LastDeleteReapAttempt = time.Now().UTC()
+		return true
+	})
+	if err != nil {
+		if errors.Is(err, ErrNodegroupNotFound) {
+			return nil
+		}
+		return err
+	}
+	if rec.Status != eks.NodegroupStatusDeleting {
+		return nil
+	}
+
+	if err := s.releaseNodegroupResources(ctx, accountID, kv, rec); err != nil {
+		if errors.Is(err, errWorkersEnding) {
+			s.noteWorkersEnding(ctx, kv, cluster, ng, err)
+			return err
+		}
+		s.noteNodegroupDeleteFailure(ctx, kv, rec, err)
+		return err
+	}
+	if err := DeleteNodegroupRecord(ctx, kv, cluster, ng); err != nil {
+		return err
+	}
+	// A cluster delete waits on its node groups; the last one going lets it run.
+	if meta, err := GetClusterMeta(ctx, kv, cluster); err == nil && meta.Status == ClusterStatusDeleting {
+		if rest, err := ListNodegroupRecords(ctx, kv, cluster); err == nil && len(rest) == 0 {
+			s.startTeardown(accountID, cluster)
+		}
+	}
+	return nil
+}
+
+// releaseNodegroupResources ends the workers, then removes the profile. The
+// profile and the recorded worker ids go only once every worker is proven
+// terminated or absent, since an accepted terminate is only a request.
+func (s *EKSServiceImpl) releaseNodegroupResources(ctx context.Context, accountID string, kv jetstream.KeyValue, rec *NodegroupRecord) error {
+	if len(rec.InstanceIDs) > 0 {
+		if err := s.deps.Worker.TerminateWorkerInstances(ctx, rec.InstanceIDs, accountID); err != nil {
+			return fmt.Errorf("terminate workers: %w", err)
+		}
+		if err := s.awaitWorkersGone(ctx, rec.InstanceIDs, accountID); err != nil {
+			return err
+		}
+		if _, err := s.mutateNodegroup(ctx, kv, rec.ClusterName, rec.Name, func(r *NodegroupRecord) bool {
+			r.InstanceIDs = nil
+			return true
+		}); err != nil {
+			return fmt.Errorf("record workers ended: %w", err)
+		}
+	}
+	return s.deleteNodegroupProfile(ctx, accountID, kv, rec)
+}
+
+// awaitWorkersGone polls until no worker is live, up to the end wait,
+// and returns errWorkersEnding if some still are.
+func (s *EKSServiceImpl) awaitWorkersGone(ctx context.Context, ids []string, accountID string) error {
+	wait, poll := s.workersEndWait, s.workersEndPoll
+	if wait <= 0 {
+		wait = defaultWorkersEndWait
+	}
+	if poll <= 0 {
+		poll = defaultWorkersEndPoll
+	}
+	deadline := time.Now().Add(wait)
+	for {
+		live, err := s.deps.Worker.LiveWorkerInstances(ctx, ids, accountID)
+		if err != nil {
+			return fmt.Errorf("confirm workers ended: %w", err)
+		}
+		if len(live) == 0 {
+			return nil
+		}
+		if !time.Now().Before(deadline) {
+			return fmt.Errorf("%w: %s", errWorkersEnding, strings.Join(live, ","))
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(poll):
+		}
+	}
+}
+
+// noteWorkersEnding handles workers that are not yet gone. Early on that is no
+// failed attempt: the next sweep checks again without backoff. Once the group
+// has been deleting past nodegroupDeleteFailAfter it settles DELETE_FAILED
+// naming the instances, keeping the profile and record for a customer retry.
+func (s *EKSServiceImpl) noteWorkersEnding(ctx context.Context, kv jetstream.KeyValue, cluster, ng string, cause error) {
+	_, err := s.mutateNodegroup(ctx, kv, cluster, ng, func(r *NodegroupRecord) bool {
+		if r.Status != eks.NodegroupStatusDeleting {
+			return false
+		}
+		if time.Since(r.DeletingSince) >= nodegroupDeleteFailAfter {
+			r.Status = eks.NodegroupStatusDeleteFailed
+			r.StatusReason = "delete: " + cause.Error()
+			r.ModifiedAt = time.Now().UTC()
+			return true
+		}
+		r.DeleteReapAttempts--
+		r.LastDeleteReapAttempt = time.Time{}
+		return true
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "eks: record workers still ending", "cluster", cluster, "nodegroup", ng, "err", err)
+	}
+}
+
+// noteNodegroupDeleteFailure records why a pass failed and, once the group has
+// been deleting long enough and failed often enough, settles it DELETE_FAILED.
+func (s *EKSServiceImpl) noteNodegroupDeleteFailure(ctx context.Context, kv jetstream.KeyValue, rec *NodegroupRecord, cause error) {
+	_, err := s.mutateNodegroup(ctx, kv, rec.ClusterName, rec.Name, func(r *NodegroupRecord) bool {
+		if r.Status != eks.NodegroupStatusDeleting {
+			return false
+		}
+		r.StatusReason = "delete: " + cause.Error()
+		r.ModifiedAt = time.Now().UTC()
+		if r.DeleteReapAttempts >= nodegroupDeleteFailAttempts && time.Since(r.DeletingSince) >= nodegroupDeleteFailAfter {
+			r.Status = eks.NodegroupStatusDeleteFailed
+		}
+		return true
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "eks: record nodegroup delete failure", "cluster", rec.ClusterName, "nodegroup", rec.Name, "err", err)
+	}
+}
@@ -379,6 +379,12 @@ func (s *EKSServiceImpl) createNodegroup(ctx context.Context, acctKV jetstream.K
 		ModifiedAt:     now,
 	}

+	// Recorded before the claim, so a name is durable before any worker launches.
+	if rec.NodeRole != "" {
+		rec.InstanceProfileID = uuid.NewV4().String()
+		rec.InstanceProfileName = nodegroupProfilePrefix + rec.InstanceProfileID
+	}
+
 	// Atomically claim the record; concurrent/duplicate CreateNodegroup requests lose here.
 	owned, err := ClaimNodegroupRecord(ctx, acctKV, rec)
 	if err != nil {
@@ -716,7 +722,7 @@ func (s *EKSServiceImpl) launchOneWorker(ctx context.Context, rec *NodegroupReco
 	// Attach the node role's instance profile so IMDS serves the role to the ECR
 	// credential provider; without it worker pulls from the internal ECR get 401.
 	if s.iamEnsurer() != nil && rec.NodeRole != "" {
-		profileARN, perr := s.ensureNodeInstanceProfile(accountID, rec.NodeRole)
+		profileARN, perr := s.nodeProfileForLaunch(accountID, rec)
 		if perr != nil {
 			return "", fmt.Errorf("ensure node instance profile: %w", perr)
 		}
@@ -855,14 +861,10 @@ func gatewayHostIP(gatewayURL string) string {
 	return host
 }

-// ensureNodeInstanceProfile guarantees an IAM instance profile named for the
-// nodegroup's node role exists and carries that role, returning its ARN. Workers
-// launch with this profile so IMDS serves the node role to the ECR credential
-// provider, mirroring the implicit instance profile real EKS creates for a node
-// role. Idempotent: concurrent worker launches converge on the same profile.
-func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string) (string, error) {
-	// Resolution compares the supplied ARN against the stored one, so an invented
-	// path cannot bind a profile to the role its trailing name happens to match.
+// resolveNodeRoleName returns the name of the stored role nodeRoleARN names.
+// Resolution compares the supplied ARN against the stored one, so an invented
+// path cannot bind a profile to the role its trailing name happens to match.
+func (s *EKSServiceImpl) resolveNodeRoleName(accountID, nodeRoleARN string) (string, error) {
 	_, roleName, err := auth.ResolveRoleARN(nodeRoleARN, func(account, name string) (string, error) {
 		// The profile is created under the caller's account, so a role from
 		// another account would silently bind a name that account does not own.
@@ -881,6 +883,19 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 	if err != nil {
 		return "", fmt.Errorf("node role ARN %q: %w", nodeRoleARN, err)
 	}
+	return roleName, nil
+}
+
+// ensureNodeInstanceProfile guarantees an IAM instance profile named for the
+// nodegroup's node role exists and carries that role, returning its ARN. Workers
+// launch with this profile so IMDS serves the node role to the ECR credential
+// provider, mirroring the implicit instance profile real EKS creates for a node
+// role. Idempotent: concurrent worker launches converge on the same profile.
+func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string) (string, error) {
+	roleName, err := s.resolveNodeRoleName(accountID, nodeRoleARN)
+	if err != nil {
+		return "", err
+	}
 	profileName := roleName

 	out, err := s.iamEnsurer().GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{
@@ -1018,6 +1033,9 @@ func (s *EKSServiceImpl) reconcileNodegroup(ctx context.Context, acctKV jetstrea
 			}
 			return nil, err
 		}
+		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusDeleteFailed {
+			return nil, errors.New(awserrors.ErrorEKSResourceInUse)
+		}
 		applyScalingUpdate(rec, scaling)
 		if labels != nil {
 			rec.Labels = applyLabelUpdate(rec.Labels, labels)
@@ -1227,33 +1245,49 @@ func (s *EKSServiceImpl) deleteNodegroup(ctx context.Context, acctKV jetstream.K
 	if err := requireCluster(ctx, acctKV, cluster); err != nil {
 		return nil, err
 	}
-	rec, err := GetNodegroupRecord(ctx, acctKV, cluster, ng)
+	rec, err := s.markNodegroupDeleting(ctx, acctKV, cluster, ng)
 	if err != nil {
-		if errors.Is(err, ErrNodegroupNotFound) {
-			return nil, nodegroupNotFound(ng)
-		}
 		return nil, err
 	}
-
-	if len(rec.InstanceIDs) > 0 {
-		if err := s.deps.Worker.TerminateWorkerInstances(ctx, rec.InstanceIDs, accountID); err != nil {
-			return nil, fmt.Errorf("terminate workers: %w", err)
-		}
+	if !s.noDeleteKick {
+		s.startNodegroupDelete(accountID, acctKV, cluster, ng)
 	}
-	if err := DeleteNodegroupRecord(ctx, acctKV, cluster, ng); err != nil {
-		return nil, err
-	}
-	// A cluster delete waits on its node groups; the last one going lets it run.
-	if meta, err := GetClusterMeta(ctx, acctKV, cluster); err == nil && meta.Status == ClusterStatusDeleting {
-		if rest, err := ListNodegroupRecords(ctx, acctKV, cluster); err == nil && len(rest) == 0 {
-			s.startTeardown(accountID, cluster)
-		}
-	}
-
-	rec.Status = eks.NodegroupStatusDeleting
 	return &eks.DeleteNodegroupOutput{Nodegroup: nodegroupRecordToAWS(rec)}, nil
 }

+// markNodegroupDeleting records DELETING and returns. The workers, the profile
+// and the record itself are the nodegroup reaper's to remove, so the request
+// never waits on work that scales with the group (a launching group is refused).
+func (s *EKSServiceImpl) markNodegroupDeleting(ctx context.Context, kv jetstream.KeyValue, cluster, ng string) (*NodegroupRecord, error) {
+	for range ngCASMaxRetries {
+		rec, rev, err := getNodegroupEntry(ctx, kv, cluster, ng)
+		if err != nil {
+			if errors.Is(err, ErrNodegroupNotFound) {
+				return nil, nodegroupNotFound(ng)
+			}
+			return nil, err
+		}
+		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusCreating {
+			return nil, errors.New(awserrors.ErrorEKSResourceInUse)
+		}
+		now := time.Now().UTC()
+		rec.Status = eks.NodegroupStatusDeleting
+		rec.StatusReason = ""
+		rec.DeletingSince = now
+		rec.DeleteReapAttempts = 0
+		rec.LastDeleteReapAttempt = time.Time{}
+		rec.ModifiedAt = now
+		ok, err := s.casPutNodegroup(ctx, kv, rec, rev)
+		if err != nil {
+			return nil, err
+		}
+		if ok {
+			return rec, nil
+		}
+	}
+	return nil, errors.New(awserrors.ErrorServerInternal)
+}
+
 // decryptNodeToken reads + decrypts the cluster's K3s join token from KV.
 func (s *EKSServiceImpl) decryptNodeToken(ctx context.Context, kv jetstream.KeyValue, cluster string) (string, error) {
 	if kv == nil {
@@ -1496,8 +1530,12 @@ func nodegroupRecordToAWS(rec *NodegroupRecord) *eks.Nodegroup {
 	// wrong; a nil slice here would marshal the key away entirely.
 	issues := make([]*eks.Issue, 0, 1)
 	if rec.StatusReason != "" {
+		code := eks.NodegroupIssueCodeNodeCreationFailure
+		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusDeleteFailed {
+			code = eks.NodegroupIssueCodeNodeTerminationFailure
+		}
 		issues = append(issues, &eks.Issue{
-			Code:        aws.String(eks.NodegroupIssueCodeNodeCreationFailure),
+			Code:        aws.String(code),
 			Message:     aws.String(rec.StatusReason),
 			ResourceIds: aws.StringSlice(rec.InstanceIDs),
 		})
@@ -0,0 +1,255 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func (f *fakeWorkerLauncher) terminateCallCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return len(f.terminateCalls)
+}
+
+func requireNodegroupStatus(t *testing.T, f *eksServiceFixture, cluster, ng, want string) *NodegroupRecord {
+	t.Helper()
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, cluster, ng)
+	require.NoError(t, err)
+	require.Equal(t, want, rec.Status)
+	return rec
+}
+
+func TestNodegroupDeletingReaper_SweepWithoutOrchestrationDepsStartsNothing(t *testing.T) {
+	t.Parallel()
+	n, err := (&EKSServiceImpl{}).NewNodegroupDeletingReaper().Sweep(t.Context())
+	require.NoError(t, err)
+	assert.Zero(t, n)
+}
+
+func TestNodegroupDeletingReaper_CancelledSweepLeavesTheDeleteRecorded(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	n, err := f.svc.NewNodegroupDeletingReaper().Sweep(ctx)
+	f.svc.WaitNodegroupDeletes()
+
+	require.ErrorIs(t, err, context.Canceled)
+	assert.Zero(t, n)
+	requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusDeleting)
+}
+
+func TestNodegroupDeletingReaper_OnlyDeletingGroupsPastBackoffAreStarted(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "active", 1)
+	createReadyNodegroup(t, f, "c1", "backoff", 1)
+	_, err := deleteNodegroupCall(f, "c1", "backoff")
+	require.NoError(t, err)
+	rec := requireNodegroupStatus(t, f, "c1", "backoff", eks.NodegroupStatusDeleting)
+	rec.DeleteReapAttempts = 3
+	rec.LastDeleteReapAttempt = time.Now().UTC()
+	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, rec))
+
+	assert.Zero(t, sweepNodegroups(t, f), "an ACTIVE group and a DELETING one inside its backoff are not due")
+	requireNodegroupStatus(t, f, "c1", "active", eks.NodegroupStatusActive)
+	assert.Zero(t, f.worker.terminateCallCount())
+}
+
+func TestConvergeNodegroupDeletion_MissingOrNotDeletingRecordIsLeftAlone(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+
+	require.NoError(t, f.svc.convergeNodegroupDeletion(t.Context(), testAccountID, f.kv, "c1", "never-existed"))
+	require.NoError(t, f.svc.convergeNodegroupDeletion(t.Context(), testAccountID, f.kv, "c1", "ng1"))
+
+	requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusActive)
+	assert.Zero(t, f.worker.terminateCallCount(), "a group nobody asked to delete keeps its workers")
+	assert.Contains(t, fixtureIAM(t, f).Profiles, mustProfileName(t, f, "c1", "ng1"))
+}
+
+func mustProfileName(t *testing.T, f *eksServiceFixture, cluster, ng string) string {
+	t.Helper()
+	rec, _ := ownedProfile(t, f, cluster, ng)
+	return rec.InstanceProfileName
+}
+
+func TestAwaitWorkersGone_CancelledWhileWorkersEndingReturnsTheContextError(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	f.worker.stillLive = []string{"i-slow"}
+	f.svc.workersEndWait = time.Hour
+	f.svc.workersEndPoll = time.Hour
+	ctx, cancel := context.WithCancel(t.Context())
+	time.AfterFunc(50*time.Millisecond, cancel)
+
+	err := f.svc.awaitWorkersGone(ctx, []string{"i-slow"}, testAccountID)
+
+	require.ErrorIs(t, err, context.Canceled)
+	assert.NotErrorIs(t, err, errWorkersEnding, "a cancelled wait is not a verdict on the workers")
+}
+
+func TestDeleteNodegroup_IAMReadFailureKeepsGroupDeletingAndProfileIntact(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	m.GetInstanceProfileErr = errors.New("iam read timed out")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	cur := requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusDeleting)
+	assert.Contains(t, cur.StatusReason, "get instance profile", "the failed read is recorded for the customer")
+	assert.Contains(t, m.Profiles, rec.InstanceProfileName, "an unreadable profile is not a missing one")
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+
+	m.GetInstanceProfileErr = nil
+	backdateNodegroupReap(t, f, "c1", "ng1")
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+}
+
+func TestDeleteNodegroup_ProfileAlreadyGoneStillFinishesTheGroup(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	delete(m.Profiles, rec.InstanceProfileName)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.Zero(t, m.RemoveRoleCalls)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+}
+
+func TestDeleteNodegroup_ProfileVanishingMidDeleteStillFinishesTheGroup(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	m := fixtureIAM(t, f)
+	m.DeleteInstanceProfileErr = awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile gone.")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.Equal(t, 1, m.RemoveRoleCalls)
+}
+
+func TestDeleteNodegroup_DeleteProfileFailureKeepsGroupDeletingUntilItClears(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	m.DeleteInstanceProfileErr = errors.New("iam unavailable")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+	cur := requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusDeleting)
+	assert.Contains(t, cur.StatusReason, "delete instance profile")
+	assert.Empty(t, cur.InstanceIDs, "workers proven gone are not terminated again")
+
+	m.DeleteInstanceProfileErr = nil
+	backdateNodegroupReap(t, f, "c1", "ng1")
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+}
+
+func TestEnsureNodegroupProfile_RecreatesAMissingRecordedProfileAsItsOwn(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	delete(m.Profiles, rec.InstanceProfileName)
+
+	arn, err := f.svc.ensureNodegroupProfile(testAccountID, rec)
+
+	require.NoError(t, err)
+	assert.Contains(t, arn, rec.InstanceProfileName)
+	got := m.Profiles[rec.InstanceProfileName]
+	require.NotNil(t, got)
+	assert.True(t, profileOwnedBy(got, rec), "a re-created profile carries this incarnation's tags")
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(got))
+}
+
+func TestEnsureNodegroupProfile_RefusesAProfileThatIsNotThisGroups(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, p := ownedProfile(t, f, "c1", "ng1")
+	p.Tags = nil
+	callsBefore := fixtureIAM(t, f).AddRoleToInstanceProfileCalls
+
+	_, err := f.svc.ensureNodegroupProfile(testAccountID, rec)
+
+	require.ErrorContains(t, err, "is not this nodegroup's")
+	assert.Equal(t, callsBefore, fixtureIAM(t, f).AddRoleToInstanceProfileCalls, "no role is attached to a profile EKS does not own")
+}
+
+func TestEnsureNodegroupProfile_IAMFailuresSurfaceWithoutHalfWork(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+
+	m.GetInstanceProfileErr = errors.New("iam read failed")
+	_, err := f.svc.ensureNodegroupProfile(testAccountID, rec)
+	require.ErrorContains(t, err, "get instance profile")
+	m.GetInstanceProfileErr = nil
+
+	delete(m.Profiles, rec.InstanceProfileName)
+	m.CreateInstanceProfileErr = errors.New("iam write failed")
+	_, err = f.svc.ensureNodegroupProfile(testAccountID, rec)
+	require.ErrorContains(t, err, "create instance profile")
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+}
+
+func TestMarkNodegroupDeleting_UnknownGroupIsResourceNotFound(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+
+	_, err := f.svc.markNodegroupDeleting(t.Context(), f.kv, "c1", "ghost")
+
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}
@@ -0,0 +1,662 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/aws/aws-sdk-go/service/iam"
+	iammock "github.com/mulgadc/spinifex/spinifex/domains/iam/mock"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	profileClusterTag   = "spinifex:eks-cluster"
+	profileNodegroupTag = "spinifex:eks-nodegroup"
+	fixtureNodeRoleName = "eks-node"
+)
+
+func fixtureIAM(t *testing.T, f *eksServiceFixture) *iammock.SystemInstanceRoleEnsurer {
+	t.Helper()
+	m, ok := f.svc.deps.IAM.(*iammock.SystemInstanceRoleEnsurer)
+	require.True(t, ok, "the fixture IAM must be the in-memory mock")
+	return m
+}
+
+// createReadyNodegroup creates a nodegroup whose n workers register Ready.
+func createReadyNodegroup(t *testing.T, f *eksServiceFixture, cluster, ng string, n int64) {
+	t.Helper()
+	markWorkersReady(t, f, cluster, ng, int(n))
+	_, err := f.svc.CreateNodegroup(context.Background(), createNGInput(cluster, ng, n), testAccountID)
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+}
+
+// createUnreadyNodegroup creates a nodegroup that never registers, so it ends
+// CREATE_FAILED holding the workers it launched.
+func createUnreadyNodegroup(t *testing.T, f *eksServiceFixture, cluster, ng string, n int64) {
+	t.Helper()
+	_, err := f.svc.CreateNodegroup(context.Background(), createNGInput(cluster, ng, n), testAccountID)
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+}
+
+func deleteNodegroupCall(f *eksServiceFixture, cluster, ng string) (*eks.DeleteNodegroupOutput, error) {
+	return f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
+		ClusterName: aws.String(cluster), NodegroupName: aws.String(ng),
+	}, testAccountID)
+}
+
+func describeNodegroupCall(f *eksServiceFixture, cluster, ng string) (*eks.DescribeNodegroupOutput, error) {
+	return f.svc.DescribeNodegroup(context.Background(), &eks.DescribeNodegroupInput{
+		ClusterName: aws.String(cluster), NodegroupName: aws.String(ng),
+	}, testAccountID)
+}
+
+func sweepNodegroups(t *testing.T, f *eksServiceFixture) int {
+	t.Helper()
+	n, err := f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
+	require.NoError(t, err)
+	f.svc.WaitNodegroupDeletes()
+	return n
+}
+
+// backdateNodegroupReap moves the last reap pass far enough back that any
+// retry backoff has elapsed.
+func backdateNodegroupReap(t *testing.T, f *eksServiceFixture, cluster, ng string) {
+	t.Helper()
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, cluster, ng)
+	require.NoError(t, err)
+	rec.LastDeleteReapAttempt = time.Now().UTC().Add(-24 * time.Hour)
+	rec.DeletingSince = time.Now().UTC().Add(-24 * time.Hour)
+	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, rec))
+}
+
+func requireErrorCode(t *testing.T, err error, want string) {
+	t.Helper()
+	require.Error(t, err)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok, "error %v carries no AWS code", err)
+	assert.Equal(t, want, code)
+}
+
+func profileTag(p *iam.InstanceProfile, key string) string {
+	for _, tg := range p.Tags {
+		if aws.StringValue(tg.Key) == key {
+			return aws.StringValue(tg.Value)
+		}
+	}
+	return ""
+}
+
+func profileHasTagValue(p *iam.InstanceProfile, value string) bool {
+	for _, tg := range p.Tags {
+		if aws.StringValue(tg.Value) == value {
+			return true
+		}
+	}
+	return false
+}
+
+func profileRoleNames(p *iam.InstanceProfile) []string {
+	var names []string
+	for _, r := range p.Roles {
+		names = append(names, aws.StringValue(r.RoleName))
+	}
+	return names
+}
+
+// ownedProfile returns the nodegroup's record and the instance profile it
+// names, failing the test if either is missing.
+func ownedProfile(t *testing.T, f *eksServiceFixture, cluster, ng string) (*NodegroupRecord, *iam.InstanceProfile) {
+	t.Helper()
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, cluster, ng)
+	require.NoError(t, err)
+	require.NotEmpty(t, rec.InstanceProfileName, "the record must name the profile it owns")
+	require.NotEmpty(t, rec.InstanceProfileID, "the record must carry the per-incarnation id")
+	p, ok := fixtureIAM(t, f).Profiles[rec.InstanceProfileName]
+	require.True(t, ok, "profile %q must exist", rec.InstanceProfileName)
+	return rec, p
+}
+
+func seedCustomerProfile(f *eksServiceFixture, t *testing.T, name, role string, tags map[string]string) {
+	t.Helper()
+	p := &iam.InstanceProfile{
+		InstanceProfileName: aws.String(name),
+		Arn:                 aws.String("arn:aws:iam::" + testAccountID + ":instance-profile/" + name),
+	}
+	if role != "" {
+		p.Roles = []*iam.Role{{RoleName: aws.String(role)}}
+	}
+	for k, v := range tags {
+		p.Tags = append(p.Tags, &iam.Tag{Key: aws.String(k), Value: aws.String(v)})
+	}
+	fixtureIAM(t, f).Profiles[name] = p
+}
+
+func TestDeleteNodegroup_ReturnsDeletingWithoutTerminatingInRequest(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+
+	out, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleting, aws.StringValue(out.Nodegroup.Status))
+
+	assert.Zero(t, f.worker.terminatedCount(), "the request must not wait for workers to end")
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err, "the record stays until the reaper has finished")
+	assert.Equal(t, eks.NodegroupStatusDeleting, rec.Status)
+
+	desc, err := describeNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleting, aws.StringValue(desc.Nodegroup.Status))
+}
+
+func TestDeleteNodegroup_ReaperEndsWorkersThenRemovesGroup(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, 1, sweepNodegroups(t, f))
+
+	require.Len(t, f.worker.terminateCalls, 1)
+	assert.ElementsMatch(t, rec.InstanceIDs, f.worker.terminateCalls[0])
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	_, ok := fixtureIAM(t, f).Profiles[rec.InstanceProfileName]
+	assert.False(t, ok, "the group is not gone until its profile is")
+}
+
+func TestDeleteNodegroup_RepeatedDeleteOnDeletingGroupIsResourceInUse(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	_, err = deleteNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
+}
+
+func TestDeleteNodegroup_DeletingGroupRefusesUpdates(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	_, err = f.svc.UpdateNodegroupConfig(context.Background(), &eks.UpdateNodegroupConfigInput{
+		ClusterName:   aws.String("c1"),
+		NodegroupName: aws.String("ng1"),
+		ScalingConfig: &eks.NodegroupScalingConfig{MinSize: aws.Int64(1), MaxSize: aws.Int64(3), DesiredSize: aws.Int64(3)},
+	}, testAccountID)
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
+	assert.Equal(t, 1, f.worker.runCount(), "a DELETING group must not launch workers")
+}
+
+func TestDeleteNodegroup_SameNameCreateWhileDeletingIsResourceInUse(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	_, err = f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
+}
+
+func TestDeleteNodegroup_DeleteClusterRefusedWhileGroupDeleting(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	_, err = f.svc.DeleteCluster(context.Background(), &eks.DeleteClusterInput{Name: aws.String("c1")}, testAccountID)
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
+}
+
+func TestDeleteNodegroup_BootReclaimLeavesDeletingGroupToReaper(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{
+		ClusterName: "c1", Name: "ng1", Status: eks.NodegroupStatusDeleting,
+		InstanceIDs: []string{"i-w1", "i-w2"}, NodeRole: fixtureNodeRoleARN,
+	}))
+
+	f.svc.reclaimOrphanedNodegroups(t.Context(), testAccountID, f.kv, "c1")
+
+	assert.Zero(t, f.worker.terminatedCount())
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleting, rec.Status)
+	assert.Equal(t, []string{"i-w1", "i-w2"}, rec.InstanceIDs)
+}
+
+func TestDeleteNodegroup_RepeatedReaperFailureEndsDeleteFailedWithHealthIssue(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	f.worker.termErr = errors.New("terminate refused")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	var rec *NodegroupRecord
+	for range 30 {
+		sweepNodegroups(t, f)
+		rec, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+		require.NoError(t, err, "a failing delete must leave the record, never drop it")
+		if rec.Status == eks.NodegroupStatusDeleteFailed {
+			break
+		}
+		backdateNodegroupReap(t, f, "c1", "ng1")
+	}
+	require.Equal(t, eks.NodegroupStatusDeleteFailed, rec.Status)
+
+	desc, err := describeNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	require.NotEmpty(t, desc.Nodegroup.Health.Issues, "DELETE_FAILED must say why")
+	assert.Equal(t, eks.NodegroupIssueCodeNodeTerminationFailure, aws.StringValue(desc.Nodegroup.Health.Issues[0].Code))
+
+	// The customer retries the delete once the cause is gone.
+	f.worker.termErr = nil
+	_, err = deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}
+
+func TestCreateNodegroup_OwnsATaggedProfileRecordedBeforeFirstLaunch(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	var seenAtLaunch string
+	f.worker.beforeRun = func(in *ec2.RunInstancesInput) {
+		if rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1"); err == nil && seenAtLaunch == "" {
+			seenAtLaunch = rec.InstanceProfileName
+		}
+	}
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+
+	rec, p := ownedProfile(t, f, "c1", "ng1")
+	assert.NotEqual(t, fixtureNodeRoleName, rec.InstanceProfileName, "the profile is the group's own, not the role-named one")
+	assert.Equal(t, rec.InstanceProfileName, seenAtLaunch, "the name must be durable before any worker launches")
+	assert.Equal(t, "c1", profileTag(p, profileClusterTag))
+	assert.Equal(t, "ng1", profileTag(p, profileNodegroupTag))
+	assert.True(t, profileHasTagValue(p, rec.InstanceProfileID), "a tag must carry the incarnation id")
+	for _, tg := range p.Tags {
+		assert.True(t, strings.HasPrefix(aws.StringValue(tg.Key), "spinifex:"), "tag %q: aws: keys are reserved, ownership tags must be spinifex:", aws.StringValue(tg.Key))
+	}
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(p))
+
+	require.Equal(t, 2, f.worker.runCount())
+	for _, in := range f.worker.runCalls {
+		require.NotNil(t, in.IamInstanceProfile)
+		assert.True(t, strings.HasSuffix(aws.StringValue(in.IamInstanceProfile.Arn), "/"+rec.InstanceProfileName),
+			"every worker launches with the group's own profile, got %s", aws.StringValue(in.IamInstanceProfile.Arn))
+	}
+}
+
+func TestCreateNodegroup_TwoGroupsOnOneRoleGetTwoProfiles(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createUnreadyNodegroup(t, f, "c1", "ng-a", 1)
+	createUnreadyNodegroup(t, f, "c1", "ng-b", 1)
+
+	recA, pA := ownedProfile(t, f, "c1", "ng-a")
+	recB, pB := ownedProfile(t, f, "c1", "ng-b")
+	assert.NotEqual(t, recA.InstanceProfileName, recB.InstanceProfileName)
+	assert.NotEqual(t, recA.InstanceProfileID, recB.InstanceProfileID)
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(pA))
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(pB))
+}
+
+func TestCreateNodegroup_RecreatedGroupOfSameNameGetsANewProfile(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createUnreadyNodegroup(t, f, "c1", "ng1", 1)
+	first, _ := ownedProfile(t, f, "c1", "ng1")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	createUnreadyNodegroup(t, f, "c1", "ng1", 1)
+	second, _ := ownedProfile(t, f, "c1", "ng1")
+	assert.NotEqual(t, first.InstanceProfileName, second.InstanceProfileName)
+	assert.NotEqual(t, first.InstanceProfileID, second.InstanceProfileID)
+	_, stillThere := fixtureIAM(t, f).Profiles[first.InstanceProfileName]
+	assert.False(t, stillThere, "the first incarnation's profile must have gone with it")
+}
+
+func TestDeleteNodegroup_AfterCreateFailedRemovesProfileAndGroup(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createUnreadyNodegroup(t, f, "c1", "ng1", 2)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	require.Equal(t, eks.NodegroupStatusCreateFailed, rec.Status)
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	_, ok := fixtureIAM(t, f).Profiles[rec.InstanceProfileName]
+	assert.False(t, ok)
+	assert.Equal(t, 2, f.worker.terminatedCount())
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}
+
+func TestDeleteNodegroup_ProfileSurvivesWhileWorkersAreLive(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	f.worker.termErr = errors.New("terminate refused")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	m := fixtureIAM(t, f)
+	assert.Contains(t, m.Profiles, rec.InstanceProfileName, "a profile in use by live workers must not be touched")
+	assert.Zero(t, m.RemoveRoleCalls)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+}
+
+func TestDeleteNodegroup_PartialProfileDeleteConvergesOnRetry(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	m.RemoveRoleFromInstanceProfileErr = errors.New("iam unavailable")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+	still, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err, "the group is not gone while its profile is")
+	assert.Equal(t, eks.NodegroupStatusDeleting, still.Status)
+
+	m.RemoveRoleFromInstanceProfileErr = nil
+	backdateNodegroupReap(t, f, "c1", "ng1")
+	sweepNodegroups(t, f)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}
+
+func TestDeleteNodegroup_CustomerRoleNamedProfileSurvives(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	seedCustomerProfile(f, t, fixtureNodeRoleName, fixtureNodeRoleName, nil)
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	m := fixtureIAM(t, f)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName, "the group's own profile goes")
+	require.Contains(t, m.Profiles, fixtureNodeRoleName, "a customer's profile is never EKS's to delete")
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[fixtureNodeRoleName]))
+	assert.Equal(t, 1, m.DeleteInstanceProfileCalls)
+}
+
+func TestDeleteNodegroup_RecordedProfileMissingOwnershipTagSurvives(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	_, err := m.UntagInstanceProfile(testAccountID, &iam.UntagInstanceProfileInput{
+		InstanceProfileName: aws.String(rec.InstanceProfileName),
+		TagKeys:             aws.StringSlice([]string{profileClusterTag}),
+	})
+	require.NoError(t, err)
+
+	_, err = deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	require.Contains(t, m.Profiles, rec.InstanceProfileName)
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[rec.InstanceProfileName]))
+	assert.Zero(t, m.RemoveRoleCalls)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}
+
+func TestDeleteNodegroup_OldLayoutGroupDeletesAndLeavesRoleNamedProfile(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	seedCustomerProfile(f, t, fixtureNodeRoleName, fixtureNodeRoleName, nil)
+	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{
+		ClusterName: "c1", Name: "old", Status: eks.NodegroupStatusActive,
+		InstanceIDs: []string{"i-old1"}, NodeRole: fixtureNodeRoleARN,
+	}))
+
+	_, err := deleteNodegroupCall(f, "c1", "old")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	require.Len(t, f.worker.terminateCalls, 1)
+	assert.Equal(t, []string{"i-old1"}, f.worker.terminateCalls[0])
+	_, err = describeNodegroupCall(f, "c1", "old")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	m := fixtureIAM(t, f)
+	require.Contains(t, m.Profiles, fixtureNodeRoleName)
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[fixtureNodeRoleName]))
+	assert.Zero(t, m.RemoveRoleCalls)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+}
+
+// No path deletes a profile unless it is recorded, tagged for this
+// incarnation, holds the record's node role and is named by no other record.
+func TestInvariant_NodegroupDeleteNeverRemovesAProfileItDoesNotOwn(t *testing.T) {
+	t.Parallel()
+	ownership := map[string]string{profileClusterTag: "c1", profileNodegroupTag: "ng1"}
+
+	t.Run("customer pre-creates a profile at a guessed name", func(t *testing.T) {
+		t.Parallel()
+		f := newEKSServiceFixture(t)
+		seedActiveClusterWithToken(t, f, "c1")
+		guesses := []string{
+			"eks-" + arn.EKSNodegroupDiscriminator(testAccountID, "c1", "ng1"),
+			"eks-c1-ng1",
+			"c1-ng1",
+			fixtureNodeRoleName,
+		}
+		for _, name := range guesses {
+			seedCustomerProfile(f, t, name, fixtureNodeRoleName, ownership)
+		}
+		createReadyNodegroup(t, f, "c1", "ng1", 1)
+		rec, _ := ownedProfile(t, f, "c1", "ng1")
+		require.NotContains(t, guesses, rec.InstanceProfileName, "the name must not be guessable")
+
+		_, err := deleteNodegroupCall(f, "c1", "ng1")
+		require.NoError(t, err)
+		sweepNodegroups(t, f)
+
+		m := fixtureIAM(t, f)
+		for _, name := range guesses {
+			require.Contains(t, m.Profiles, name, "customer profile %q was deleted", name)
+			assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[name]), "customer profile %q lost its role", name)
+		}
+	})
+
+	t.Run("recorded profile was swapped for one holding another role", func(t *testing.T) {
+		t.Parallel()
+		f := newEKSServiceFixture(t)
+		seedActiveClusterWithToken(t, f, "c1")
+		createReadyNodegroup(t, f, "c1", "ng1", 1)
+		rec, p := ownedProfile(t, f, "c1", "ng1")
+		p.Roles = []*iam.Role{{RoleName: aws.String("customer-role")}}
+
+		_, err := deleteNodegroupCall(f, "c1", "ng1")
+		require.NoError(t, err)
+		sweepNodegroups(t, f)
+
+		m := fixtureIAM(t, f)
+		require.Contains(t, m.Profiles, rec.InstanceProfileName)
+		assert.Equal(t, []string{"customer-role"}, profileRoleNames(m.Profiles[rec.InstanceProfileName]))
+		assert.Zero(t, m.RemoveRoleCalls)
+		assert.Zero(t, m.DeleteInstanceProfileCalls)
+	})
+
+	t.Run("another nodegroup record names the same profile", func(t *testing.T) {
+		t.Parallel()
+		f := newEKSServiceFixture(t)
+		seedActiveClusterWithToken(t, f, "c1")
+		createReadyNodegroup(t, f, "c1", "ng1", 1)
+		rec, _ := ownedProfile(t, f, "c1", "ng1")
+		require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{
+			ClusterName: "c1", Name: "ng2", Status: eks.NodegroupStatusActive, NodeRole: fixtureNodeRoleARN,
+			InstanceProfileName: rec.InstanceProfileName, InstanceProfileID: rec.InstanceProfileID,
+		}))
+
+		_, err := deleteNodegroupCall(f, "c1", "ng1")
+		require.NoError(t, err)
+		sweepNodegroups(t, f)
+
+		m := fixtureIAM(t, f)
+		require.Contains(t, m.Profiles, rec.InstanceProfileName)
+		assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[rec.InstanceProfileName]))
+		assert.Zero(t, m.DeleteInstanceProfileCalls)
+	})
+}
+
+func TestDeleteNodegroup_ProfileAndRecordKeptUntilWorkersProvenGone(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	f.worker.stillLive = []string{rec.InstanceIDs[0]}
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	m := fixtureIAM(t, f)
+	assert.Contains(t, m.Profiles, rec.InstanceProfileName)
+	assert.Zero(t, m.RemoveRoleCalls)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+	got, err := describeNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleting, aws.StringValue(got.Nodegroup.Status))
+	cur, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, rec.InstanceIDs, cur.InstanceIDs, "worker ids stay recorded until they are proven gone")
+	assert.Zero(t, cur.DeleteReapAttempts, "workers still ending is not a failed pass")
+
+	f.worker.mu.Lock()
+	f.worker.stillLive = nil
+	f.worker.mu.Unlock()
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+}
+
+func TestDeleteNodegroup_UndecidableWorkerStateKeepsProfile(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	f.worker.liveErr = errors.New("describe fan-out did not complete")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	m := fixtureIAM(t, f)
+	assert.Contains(t, m.Profiles, rec.InstanceProfileName)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+}
+
+func TestDeleteNodegroup_KickFinishesTheDeleteWithoutAReaperSweep(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	f.svc.noDeleteKick = false
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	f.svc.WaitNodegroupDeletes()
+
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
+}
+
+func TestDeleteNodegroup_WorkerStuckPastDeadlineEndsDeleteFailedKeepingProfile(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	stuck := rec.InstanceIDs[0]
+	f.worker.stillLive = []string{stuck}
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+	cur, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err)
+	require.Equal(t, eks.NodegroupStatusDeleting, cur.Status, "within the deadline the group keeps waiting")
+
+	backdateNodegroupReap(t, f, "c1", "ng1")
+	sweepNodegroups(t, f)
+
+	desc, err := describeNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleteFailed, aws.StringValue(desc.Nodegroup.Status))
+	require.NotEmpty(t, desc.Nodegroup.Health.Issues)
+	issue := desc.Nodegroup.Health.Issues[0]
+	assert.Equal(t, eks.NodegroupIssueCodeNodeTerminationFailure, aws.StringValue(issue.Code))
+	assert.Contains(t, aws.StringValue(issue.Message), stuck)
+	assert.Contains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
+
+	f.worker.mu.Lock()
+	f.worker.stillLive = nil
+	f.worker.mu.Unlock()
+	_, err = deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
+}
@@ -0,0 +1,174 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+const (
+	nodegroupProfilePrefix = "eks-"
+
+	profileClusterTagKey   = "spinifex:eks-cluster"
+	profileNodegroupTagKey = "spinifex:eks-nodegroup"
+	profileIDTagKey        = "spinifex:eks-profile-id"
+)
+
+// nodeProfileForLaunch returns the ARN of the instance profile a worker of rec
+// launches with. A record that names no profile of its own predates per-group
+// profiles and keeps the role-named one it already uses.
+func (s *EKSServiceImpl) nodeProfileForLaunch(accountID string, rec *NodegroupRecord) (string, error) {
+	if rec.InstanceProfileName == "" {
+		return s.ensureNodeInstanceProfile(accountID, rec.NodeRole)
+	}
+	return s.ensureNodegroupProfile(accountID, rec)
+}
+
+func nodegroupProfileTags(rec *NodegroupRecord) []*iam.Tag {
+	return []*iam.Tag{
+		{Key: aws.String(profileClusterTagKey), Value: aws.String(rec.ClusterName)},
+		{Key: aws.String(profileNodegroupTagKey), Value: aws.String(rec.Name)},
+		{Key: aws.String(profileIDTagKey), Value: aws.String(rec.InstanceProfileID)},
+	}
+}
+
+// profileOwnedBy reports whether p carries the ownership tags of this exact
+// incarnation of rec.
+func profileOwnedBy(p *iam.InstanceProfile, rec *NodegroupRecord) bool {
+	have := map[string]string{}
+	for _, t := range p.Tags {
+		have[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
+	}
+	for _, want := range nodegroupProfileTags(rec) {
+		if have[aws.StringValue(want.Key)] != aws.StringValue(want.Value) {
+			return false
+		}
+	}
+	return true
+}
+
+func roleNameFromARN(roleARN string) string {
+	return roleARN[strings.LastIndex(roleARN, "/")+1:]
+}
+
+// ensureNodegroupProfile finds or creates the instance profile recorded for
+// rec, tagged as its own, and makes sure it holds the node role. Concurrent
+// worker launches converge on the one profile.
+func (s *EKSServiceImpl) ensureNodegroupProfile(accountID string, rec *NodegroupRecord) (string, error) {
+	roleName, err := s.resolveNodeRoleName(accountID, rec.NodeRole)
+	if err != nil {
+		return "", err
+	}
+	name := rec.InstanceProfileName
+	ens := s.iamEnsurer()
+
+	got, err := ens.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String(name)})
+	if err != nil {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
+			return "", fmt.Errorf("get instance profile %q: %w", name, err)
+		}
+		_, cerr := ens.CreateInstanceProfile(accountID, &iam.CreateInstanceProfileInput{
+			InstanceProfileName: aws.String(name),
+			Tags:                nodegroupProfileTags(rec),
+		})
+		if cerr != nil {
+			// A racing launch may have created it first; re-read and converge.
+			if code, _ := awserrors.ResolveErrorCode(cerr); code != awserrors.ErrorIAMEntityAlreadyExists {
+				return "", fmt.Errorf("create instance profile %q: %w", name, cerr)
+			}
+		}
+		got, err = ens.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String(name)})
+		if err != nil {
+			return "", fmt.Errorf("re-get instance profile %q: %w", name, err)
+		}
+	}
+	if !profileOwnedBy(got.InstanceProfile, rec) {
+		return "", fmt.Errorf("instance profile %q exists but is not this nodegroup's", name)
+	}
+	return s.attachRoleToProfile(accountID, name, roleName,
+		aws.StringValue(got.InstanceProfile.Arn), len(got.InstanceProfile.Roles) > 0)
+}
+
+// deleteNodegroupProfile removes the profile recorded for rec, and only that
+// one: it must be recorded, tagged for this incarnation, hold exactly the node
+// role and be named by no other record. Anything else is left alone.
+func (s *EKSServiceImpl) deleteNodegroupProfile(ctx context.Context, accountID string, kv jetstream.KeyValue, rec *NodegroupRecord) error {
+	name := rec.InstanceProfileName
+	ens := s.iamEnsurer()
+	if name == "" || ens == nil {
+		return nil
+	}
+	got, err := ens.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String(name)})
+	if err != nil {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
+			return nil
+		}
+		return fmt.Errorf("get instance profile %q: %w", name, err)
+	}
+	p := got.InstanceProfile
+
+	if !profileOwnedBy(p, rec) {
+		slog.WarnContext(ctx, "eks: leaving instance profile whose ownership tags do not match its nodegroup",
+			"cluster", rec.ClusterName, "nodegroup", rec.Name, "profile", name)
+		return nil
+	}
+	nodeRole := roleNameFromARN(rec.NodeRole)
+	if len(p.Roles) > 1 || (len(p.Roles) == 1 && aws.StringValue(p.Roles[0].RoleName) != nodeRole) {
+		slog.WarnContext(ctx, "eks: leaving instance profile that holds a role other than the node role",
+			"cluster", rec.ClusterName, "nodegroup", rec.Name, "profile", name)
+		return nil
+	}
+	shared, err := s.profileNamedByOtherRecord(ctx, kv, rec)
+	if err != nil {
+		return err
+	}
+	if shared {
+		slog.WarnContext(ctx, "eks: leaving instance profile another nodegroup record names",
+			"cluster", rec.ClusterName, "nodegroup", rec.Name, "profile", name)
+		return nil
+	}
+
+	if len(p.Roles) == 1 {
+		_, err := ens.RemoveRoleFromInstanceProfile(accountID, &iam.RemoveRoleFromInstanceProfileInput{
+			InstanceProfileName: aws.String(name), RoleName: aws.String(nodeRole),
+		})
+		if err != nil {
+			if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
+				return fmt.Errorf("remove role %q from instance profile %q: %w", nodeRole, name, err)
+			}
+		}
+	}
+	if _, err := ens.DeleteInstanceProfile(accountID, &iam.DeleteInstanceProfileInput{InstanceProfileName: aws.String(name)}); err != nil {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
+			return fmt.Errorf("delete instance profile %q: %w", name, err)
+		}
+	}
+	return nil
+}
+
+// profileNamedByOtherRecord reports whether any nodegroup record in the
+// account other than rec names rec's profile.
+func (s *EKSServiceImpl) profileNamedByOtherRecord(ctx context.Context, kv jetstream.KeyValue, rec *NodegroupRecord) (bool, error) {
+	clusters, err := listClusterNames(ctx, kv)
+	if err != nil {
+		return false, fmt.Errorf("list clusters: %w", err)
+	}
+	for _, cluster := range clusters {
+		recs, err := ListNodegroupRecords(ctx, kv, cluster)
+		if err != nil {
+			return false, fmt.Errorf("list nodegroups of %s: %w", cluster, err)
+		}
+		for _, other := range recs {
+			if other.InstanceProfileName == rec.InstanceProfileName && (other.ClusterName != rec.ClusterName || other.Name != rec.Name) {
+				return true, nil
+			}
+		}
+	}
+	return false, nil
+}
@@ -86,6 +86,12 @@ type fakeWorkerLauncher struct {
 	// before subsequent calls succeed — simulates a transient launch failure
 	// (QMP timeout, host pressure) that clears on retry.
 	failNextN int
+	// beforeRun, when set, runs at the start of every launch, before it is
+	// recorded, so a test can inspect durable state at launch time.
+	beforeRun func(*ec2.RunInstancesInput)
+	// stillLive lists instances LiveWorkerInstances keeps reporting as live.
+	stillLive []string
+	liveErr   error
 }

 var _ WorkerLauncher = (*fakeWorkerLauncher)(nil)
@@ -101,6 +107,9 @@ func (f *fakeWorkerLauncher) RunWorkerInstance(ctx context.Context, input *ec2.R
 func (f *fakeWorkerLauncher) RunWorkerInstanceOnNode(_ context.Context, nodeID string, input *ec2.RunInstancesInput, _ string) (*ec2.Reservation, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
+	if f.beforeRun != nil {
+		f.beforeRun(input)
+	}
 	f.runCalls = append(f.runCalls, input)
 	f.runNodes = append(f.runNodes, nodeID)
 	if f.failNextN > 0 {
@@ -129,6 +138,21 @@ func (f *fakeWorkerLauncher) TerminateWorkerInstances(_ context.Context, ids []s
 	return f.termErr
 }

+func (f *fakeWorkerLauncher) LiveWorkerInstances(_ context.Context, ids []string, _ string) ([]string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.liveErr != nil {
+		return nil, f.liveErr
+	}
+	var live []string
+	for _, id := range ids {
+		if slices.Contains(f.stillLive, id) {
+			live = append(live, id)
+		}
+	}
+	return live, nil
+}
+
 // runCount / terminatedCount return the recorded call totals under the lock so
 // concurrent tests can read them safely after joining.
 func (f *fakeWorkerLauncher) runCount() int {
@@ -927,7 +951,7 @@ func TestUpdateNodegroupConfig_CapacityErrorSurfacesCode(t *testing.T) {
 	})
 }

-func TestDeleteNodegroup_TerminatesAndIsIdempotent(t *testing.T) {
+func TestDeleteNodegroup_ReapedGroupIsGoneAndRepeatDeleteIsNotFound(t *testing.T) {
 	t.Parallel()
 	f := newEKSServiceFixture(t)
 	seedActiveClusterWithToken(t, f, "c1")
@@ -940,13 +964,16 @@ func TestDeleteNodegroup_TerminatesAndIsIdempotent(t *testing.T) {
 		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1"),
 	}, testAccountID)
 	require.NoError(t, err)
+	_, err = f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
+	require.NoError(t, err)
+	f.svc.WaitNodegroupDeletes()
 	require.Len(t, f.worker.terminateCalls, 1)
 	assert.Len(t, f.worker.terminateCalls[0], 2)

 	_, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
 	assert.ErrorIs(t, err, ErrNodegroupNotFound)

-	// Second delete → ResourceNotFoundException (record already gone).
+	// Once the reaper has finished the group is gone: ResourceNotFoundException.
 	_, err = f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
 		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1"),
 	}, testAccountID)
@@ -201,20 +201,26 @@ type WorkerLauncher interface {
 	// host spread. An empty nodeID launches on the local node like RunWorkerInstance.
 	RunWorkerInstanceOnNode(ctx context.Context, nodeID string, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error)
 	TerminateWorkerInstances(ctx context.Context, instanceIDs []string, accountID string) error
+	// LiveWorkerInstances returns the given workers not yet proven terminated or
+	// absent. An undecidable answer is an error, never an empty list.
+	LiveWorkerInstances(ctx context.Context, instanceIDs []string, accountID string) ([]string, error)
 }

 // iamEnsurer resolves the IAM ensurer, preferring the test-injected deps.IAM,
 // then the lazy IAMProvider (built at launch time to dodge the daemon-startup
 // NATS-KV race). Returns nil when neither yields a service, so callers fall back
-// to static creds / skip the profile. The provider's concrete service satisfies
-// instanceProfileEnsurer (identical method set to SystemInstanceRoleEnsurer).
+// to static creds / skip the profile.
 func (s *EKSServiceImpl) iamEnsurer() instanceProfileEnsurer {
 	if s.deps.IAM != nil {
 		return s.deps.IAM
 	}
 	if s.deps.IAMProvider != nil {
 		if e := s.deps.IAMProvider(); e != nil {
-			return e
+			// The concrete service has the wider surface; the shared interface
+			// stays narrow so other callers are not forced to grow it.
+			if ep, ok := e.(instanceProfileEnsurer); ok {
+				return ep
+			}
 		}
 	}
 	return nil
@@ -227,6 +233,8 @@ type instanceProfileEnsurer interface {
 	GetInstanceProfile(accountID string, input *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error)
 	CreateInstanceProfile(accountID string, input *iam.CreateInstanceProfileInput) (*iam.CreateInstanceProfileOutput, error)
 	AddRoleToInstanceProfile(accountID string, input *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error)
+	RemoveRoleFromInstanceProfile(accountID string, input *iam.RemoveRoleFromInstanceProfileInput) (*iam.RemoveRoleFromInstanceProfileOutput, error)
+	DeleteInstanceProfile(accountID string, input *iam.DeleteInstanceProfileInput) (*iam.DeleteInstanceProfileOutput, error)

 	// Role find-or-create surface for the system-managed control-plane role.
 	// Unlike worker node roles (customer-supplied), the k3s server role is
@@ -276,6 +284,12 @@ type EKSServiceImpl struct {
 	teardownMu      sync.Mutex
 	teardownRunning map[string]struct{}
 	teardownWG      sync.WaitGroup
+	// noDeleteKick, set by tests, leaves a nodegroup delete to the reaper sweep.
+	noDeleteKick bool
+	// workersEndWait and workersEndPoll bound how long one delete pass waits for
+	// ended workers to be proven gone; zero means the defaults.
+	workersEndWait time.Duration
+	workersEndPoll time.Duration
 	// teardownKV, when set by a test, replaces the converger's account bucket.
 	teardownKV func(ctx context.Context, accountID string) (jetstream.KeyValue, error)

@@ -49,6 +49,16 @@ type NodegroupRecord struct {
 	Tags           map[string]string `json:"tags,omitempty"`
 	InstanceIDs    []string          `json:"instanceIds,omitempty"`
 	Health         string            `json:"health,omitempty"`
+	// InstanceProfileID and InstanceProfileName identify the instance profile
+	// EKS made for this incarnation of the nodegroup; both are empty on a record
+	// that predates it.
+	InstanceProfileID   string `json:"instanceProfileId,omitempty"`
+	InstanceProfileName string `json:"instanceProfileName,omitempty"`
+	// DeletingSince, DeleteReapAttempts and LastDeleteReapAttempt pace the reaper that finishes
+	// a DELETING nodegroup.
+	DeletingSince         time.Time `json:"deletingSince,omitzero"`
+	DeleteReapAttempts    int       `json:"deleteReapAttempts,omitempty"`
+	LastDeleteReapAttempt time.Time `json:"lastDeleteReapAttempt,omitzero"`
 	// GPUEnabled/GPUVendor cache whether InstanceTypes includes a GPU family and
 	// which vendor, so worker launches resolve the matching GPU node AMI without
 	// re-scanning InstanceTypes on every launch.
@@ -56,6 +56,9 @@ func TestListManagedWorkers_DropsWorkersTheNodegroupTerminated(t *testing.T) {
 		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng2"),
 	}, testAccountID)
 	require.NoError(t, err)
+	_, err = f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
+	require.NoError(t, err)
+	f.svc.WaitNodegroupDeletes()
 	got = managedWorkers(t, f, "c1")
 	assert.Len(t, got, 1)
 	for _, id := range ng2.InstanceIDs {
@@ -36,6 +36,9 @@ type SystemInstanceRoleEnsurer struct {
 	CreateRoleCalls               int
 	CreateInstanceProfileCalls    int
 	AddRoleToInstanceProfileCalls int
+	TagInstanceProfileCalls       int
+	RemoveRoleCalls               int
+	DeleteInstanceProfileCalls    int

 	// LastTrustDoc captures the AssumeRolePolicyDocument of the most recent
 	// CreateRole call.
@@ -54,6 +57,14 @@ type SystemInstanceRoleEnsurer struct {
 	// AddRoleToInstanceProfileErr, when set, is returned by
 	// AddRoleToInstanceProfile instead of attaching.
 	AddRoleToInstanceProfileErr error
+	// RemoveRoleFromInstanceProfileErr and DeleteInstanceProfileErr, when set,
+	// are returned by the matching call instead of acting.
+	RemoveRoleFromInstanceProfileErr error
+	DeleteInstanceProfileErr         error
+	// GetInstanceProfileErr, when set, is returned by GetInstanceProfile
+	// instead of reading, for exercising callers that must tell a failed read
+	// from a missing profile.
+	GetInstanceProfileErr error
 	// EmptyInstanceProfileARN makes CreateInstanceProfile return an empty
 	// ARN, for exercising callers that must reject it.
 	EmptyInstanceProfileARN bool
@@ -81,6 +92,7 @@ func copyRole(r *iam.Role) *iam.Role {
 func copyProfile(p *iam.InstanceProfile) *iam.InstanceProfile {
 	c := *p
 	c.Roles = append([]*iam.Role(nil), p.Roles...)
+	c.Tags = append([]*iam.Tag(nil), p.Tags...)
 	return &c
 }

@@ -130,6 +142,9 @@ func (e *SystemInstanceRoleEnsurer) GetInstanceProfile(_ string, in *iam.GetInst
 	e.mu.Lock()
 	defer e.mu.Unlock()

+	if e.GetInstanceProfileErr != nil {
+		return nil, e.GetInstanceProfileErr
+	}
 	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
 	if !ok {
 		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
@@ -157,6 +172,7 @@ func (e *SystemInstanceRoleEnsurer) CreateInstanceProfile(accountID string, in *
 	p := &iam.InstanceProfile{
 		InstanceProfileName: aws.String(name),
 		Arn:                 aws.String(arn),
+		Tags:                append([]*iam.Tag(nil), in.Tags...),
 	}
 	e.Profiles[name] = p
 	return &iam.CreateInstanceProfileOutput{InstanceProfile: copyProfile(p)}, nil
@@ -181,3 +197,101 @@ func (e *SystemInstanceRoleEnsurer) AddRoleToInstanceProfile(_ string, in *iam.A
 	p.Roles = []*iam.Role{{RoleName: in.RoleName}}
 	return &iam.AddRoleToInstanceProfileOutput{}, nil
 }
+
+func (e *SystemInstanceRoleEnsurer) TagInstanceProfile(_ string, in *iam.TagInstanceProfileInput) (*iam.TagInstanceProfileOutput, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	e.TagInstanceProfileCalls++
+	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
+	if !ok {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
+	}
+	for _, t := range in.Tags {
+		replaced := false
+		for i, have := range p.Tags {
+			if aws.StringValue(have.Key) == aws.StringValue(t.Key) {
+				p.Tags[i] = t
+				replaced = true
+				break
+			}
+		}
+		if !replaced {
+			p.Tags = append(p.Tags, t)
+		}
+	}
+	return &iam.TagInstanceProfileOutput{}, nil
+}
+
+func (e *SystemInstanceRoleEnsurer) UntagInstanceProfile(_ string, in *iam.UntagInstanceProfileInput) (*iam.UntagInstanceProfileOutput, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
+	if !ok {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
+	}
+	drop := map[string]bool{}
+	for _, k := range in.TagKeys {
+		drop[aws.StringValue(k)] = true
+	}
+	kept := p.Tags[:0:0]
+	for _, t := range p.Tags {
+		if !drop[aws.StringValue(t.Key)] {
+			kept = append(kept, t)
+		}
+	}
+	p.Tags = kept
+	return &iam.UntagInstanceProfileOutput{}, nil
+}
+
+func (e *SystemInstanceRoleEnsurer) ListInstanceProfileTags(_ string, in *iam.ListInstanceProfileTagsInput) (*iam.ListInstanceProfileTagsOutput, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
+	if !ok {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
+	}
+	return &iam.ListInstanceProfileTagsOutput{Tags: append([]*iam.Tag(nil), p.Tags...)}, nil
+}
+
+func (e *SystemInstanceRoleEnsurer) RemoveRoleFromInstanceProfile(_ string, in *iam.RemoveRoleFromInstanceProfileInput) (*iam.RemoveRoleFromInstanceProfileOutput, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	e.RemoveRoleCalls++
+	if e.RemoveRoleFromInstanceProfileErr != nil {
+		return nil, e.RemoveRoleFromInstanceProfileErr
+	}
+	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
+	if !ok {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
+	}
+	if len(p.Roles) == 0 || aws.StringValue(p.Roles[0].RoleName) != aws.StringValue(in.RoleName) {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The role with name %s cannot be found.", aws.StringValue(in.RoleName))
+	}
+	p.Roles = nil
+	return &iam.RemoveRoleFromInstanceProfileOutput{}, nil
+}
+
+// DeleteInstanceProfile refuses while a role is attached, as IAM does.
+func (e *SystemInstanceRoleEnsurer) DeleteInstanceProfile(_ string, in *iam.DeleteInstanceProfileInput) (*iam.DeleteInstanceProfileOutput, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	e.DeleteInstanceProfileCalls++
+	if e.DeleteInstanceProfileErr != nil {
+		return nil, e.DeleteInstanceProfileErr
+	}
+	name := aws.StringValue(in.InstanceProfileName)
+	p, ok := e.Profiles[name]
+	if !ok {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", name)
+	}
+	if len(p.Roles) > 0 {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first.")
+	}
+	delete(e.Profiles, name)
+	return &iam.DeleteInstanceProfileOutput{}, nil
+}
@@ -83,3 +83,112 @@ func TestPutRolePolicy_InjectedErrorStillRecordsCall(t *testing.T) {
 		t.Fatalf("RolePolicies[r1] = %q, want %q", e.RolePolicies["r1"], "doc")
 	}
 }
+
+func seedProfile(t *testing.T, e *mock.SystemInstanceRoleEnsurer, name string) {
+	t.Helper()
+	if _, err := e.CreateInstanceProfile("acct", &iam.CreateInstanceProfileInput{InstanceProfileName: aws.String(name)}); err != nil {
+		t.Fatalf("CreateInstanceProfile: %v", err)
+	}
+}
+
+func requireCode(t *testing.T, err error, want string) {
+	t.Helper()
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != want {
+		t.Fatalf("err = %v, want %s", err, want)
+	}
+}
+
+func TestInstanceProfileTags_TagReplacesSameKeyUntagDropsListReturnsCopy(t *testing.T) {
+	e := mock.New()
+	seedProfile(t, e, "p1")
+	tag := func(k, v string) *iam.Tag { return &iam.Tag{Key: aws.String(k), Value: aws.String(v)} }
+	if _, err := e.TagInstanceProfile("acct", &iam.TagInstanceProfileInput{
+		InstanceProfileName: aws.String("p1"), Tags: []*iam.Tag{tag("a", "1"), tag("b", "2")},
+	}); err != nil {
+		t.Fatalf("TagInstanceProfile: %v", err)
+	}
+	if _, err := e.TagInstanceProfile("acct", &iam.TagInstanceProfileInput{
+		InstanceProfileName: aws.String("p1"), Tags: []*iam.Tag{tag("a", "9")},
+	}); err != nil {
+		t.Fatalf("re-tag: %v", err)
+	}
+	if _, err := e.UntagInstanceProfile("acct", &iam.UntagInstanceProfileInput{
+		InstanceProfileName: aws.String("p1"), TagKeys: aws.StringSlice([]string{"b"}),
+	}); err != nil {
+		t.Fatalf("UntagInstanceProfile: %v", err)
+	}
+	out, err := e.ListInstanceProfileTags("acct", &iam.ListInstanceProfileTagsInput{InstanceProfileName: aws.String("p1")})
+	if err != nil {
+		t.Fatalf("ListInstanceProfileTags: %v", err)
+	}
+	if len(out.Tags) != 1 || aws.StringValue(out.Tags[0].Key) != "a" || aws.StringValue(out.Tags[0].Value) != "9" {
+		t.Fatalf("tags = %v, want only a=9", out.Tags)
+	}
+}
+
+func TestInstanceProfileTags_MissingProfileIsNoSuchEntity(t *testing.T) {
+	e := mock.New()
+	name := aws.String("absent")
+	_, err := e.TagInstanceProfile("acct", &iam.TagInstanceProfileInput{InstanceProfileName: name})
+	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
+	_, err = e.UntagInstanceProfile("acct", &iam.UntagInstanceProfileInput{InstanceProfileName: name})
+	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
+	_, err = e.ListInstanceProfileTags("acct", &iam.ListInstanceProfileTagsInput{InstanceProfileName: name})
+	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
+}
+
+func TestDeleteInstanceProfile_RefusesWhileRoleAttachedThenDeletes(t *testing.T) {
+	e := mock.New()
+	seedProfile(t, e, "p1")
+	if _, err := e.AddRoleToInstanceProfile("acct", &iam.AddRoleToInstanceProfileInput{
+		InstanceProfileName: aws.String("p1"), RoleName: aws.String("r1"),
+	}); err != nil {
+		t.Fatalf("AddRoleToInstanceProfile: %v", err)
+	}
+	del := &iam.DeleteInstanceProfileInput{InstanceProfileName: aws.String("p1")}
+	_, err := e.DeleteInstanceProfile("acct", del)
+	requireCode(t, err, awserrors.ErrorIAMDeleteConflict)
+	if _, ok := e.Profiles["p1"]; !ok {
+		t.Fatal("a refused delete must leave the profile in place")
+	}
+
+	_, err = e.RemoveRoleFromInstanceProfile("acct", &iam.RemoveRoleFromInstanceProfileInput{
+		InstanceProfileName: aws.String("p1"), RoleName: aws.String("other"),
+	})
+	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
+	if _, err := e.RemoveRoleFromInstanceProfile("acct", &iam.RemoveRoleFromInstanceProfileInput{
+		InstanceProfileName: aws.String("p1"), RoleName: aws.String("r1"),
+	}); err != nil {
+		t.Fatalf("RemoveRoleFromInstanceProfile: %v", err)
+	}
+	if _, err := e.DeleteInstanceProfile("acct", del); err != nil {
+		t.Fatalf("DeleteInstanceProfile: %v", err)
+	}
+	if _, ok := e.Profiles["p1"]; ok {
+		t.Fatal("profile still present after delete")
+	}
+	_, err = e.DeleteInstanceProfile("acct", del)
+	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
+}
+
+func TestInstanceProfileMutations_MissingProfileAndInjectedErrors(t *testing.T) {
+	e := mock.New()
+	name := aws.String("absent")
+	_, err := e.RemoveRoleFromInstanceProfile("acct", &iam.RemoveRoleFromInstanceProfileInput{InstanceProfileName: name, RoleName: aws.String("r")})
+	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
+
+	boom := errors.New("boom")
+	e.RemoveRoleFromInstanceProfileErr, e.DeleteInstanceProfileErr, e.GetInstanceProfileErr = boom, boom, boom
+	if _, err := e.RemoveRoleFromInstanceProfile("acct", &iam.RemoveRoleFromInstanceProfileInput{InstanceProfileName: name, RoleName: aws.String("r")}); !errors.Is(err, boom) {
+		t.Fatalf("RemoveRole err = %v, want injected", err)
+	}
+	if _, err := e.DeleteInstanceProfile("acct", &iam.DeleteInstanceProfileInput{InstanceProfileName: name}); !errors.Is(err, boom) {
+		t.Fatalf("Delete err = %v, want injected", err)
+	}
+	if _, err := e.GetInstanceProfile("acct", &iam.GetInstanceProfileInput{InstanceProfileName: name}); !errors.Is(err, boom) {
+		t.Fatalf("Get err = %v, want injected", err)
+	}
+	if e.RemoveRoleCalls != 2 || e.DeleteInstanceProfileCalls != 1 {
+		t.Fatalf("calls = remove %d delete %d, want 2 and 1", e.RemoveRoleCalls, e.DeleteInstanceProfileCalls)
+	}
+}
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"log/slog"
 	"net/http"
+	"strconv"
 	"time"

 	"github.com/aws/aws-sdk-go/aws"
@@ -18,47 +19,109 @@ import (
 // codeURLTTL is how long the presigned code location stays valid.
 const codeURLTTL = 10 * time.Minute

+// maxListItems is the largest MaxItems a list action accepts.
+const maxListItems = 10000
+
 func notFound(region, accountID, name string) error {
 	return awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "Function not found: %s",
 		arn.FormatLambdaFunction(region, accountID, name))
 }

-// getFunction implements GetFunction for the $LATEST version.
-func (d Deps) getFunction(ctx context.Context, c *call) (reply, error) {
+// lookup resolves the requested $LATEST function's record; query is the
+// Qualifier query parameter, empty for an action that has none.
+func (d Deps) lookup(ctx context.Context, c *call, query string) (function.Record, uint64, error) {
 	name, q, err := resolveFunctionName(c.Region, c.AccountID, c.Params[0])
 	if err != nil {
-		return reply{}, err
+		return function.Record{}, 0, err
 	}
-	if query := c.Query.Get("Qualifier"); query != "" {
+	if query != "" {
 		if q != "" && q != query {
-			return reply{}, invalid("The derived qualifier from the function name does not match the specified qualifier.")
+			return function.Record{}, 0, invalid("The derived qualifier from the function name does not match the specified qualifier.")
 		}
 		q = query
 	}
 	if q != "" && q != "$LATEST" {
-		return reply{}, notFound(c.Region, c.AccountID, name+":"+q)
+		return function.Record{}, 0, notFound(c.Region, c.AccountID, name+":"+q)
 	}
 	rec, revision, err := d.Meta.Get(ctx, c.AccountID, c.Region, name)
 	if errors.Is(err, function.ErrNotFound) {
-		return reply{}, notFound(c.Region, c.AccountID, name)
+		return function.Record{}, 0, notFound(c.Region, c.AccountID, name)
 	}
+	return rec, revision, err
+}
+
+// loadConfiguration resolves the requested $LATEST function and renders its
+// configuration; it is a metadata read and never touches the code store.
+func (d Deps) loadConfiguration(ctx context.Context, c *call, query string) (function.Record, *lambda.FunctionConfiguration, error) {
+	rec, revision, err := d.lookup(ctx, c, query)
 	if err != nil {
-		return reply{}, err
+		return function.Record{}, nil, err
 	}

 	var env map[string]string
 	if rec.EnvCipher != "" {
 		if d.AEAD == nil {
-			return reply{}, serviceError("environment decryption is unavailable")
+			return function.Record{}, nil, serviceError("environment decryption is unavailable")
 		}
 		if env, err = function.OpenEnv(d.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher); err != nil {
 			slog.ErrorContext(ctx, "Lambda: open environment failed", "function_id", rec.FunctionID, "err", err)
-			return reply{}, serviceError("could not read the function environment")
+			return function.Record{}, nil, serviceError("could not read the function environment")
 		}
 	}
 	cfg, err := configuration(rec, revision, env)
 	if err != nil {
-		return reply{}, serviceError("could not build the function configuration")
+		return function.Record{}, nil, serviceError("could not build the function configuration")
+	}
+	return rec, cfg, nil
+}
+
+// getFunctionConfiguration implements GetFunctionConfiguration for $LATEST.
+func (d Deps) getFunctionConfiguration(ctx context.Context, c *call) (reply, error) {
+	_, cfg, err := d.loadConfiguration(ctx, c, c.Query.Get("Qualifier"))
+	if err != nil {
+		return reply{}, err
+	}
+	return reply{Status: http.StatusOK, Body: cfg}, nil
+}
+
+// noCodeSigning is AWS's body for a function without a code signing config:
+// both members present and null, which the SDK shapes cannot encode.
+var noCodeSigning = rawJSON(`{"CodeSigningConfigArn":null,"FunctionName":null}`)
+
+// getFunctionCodeSigningConfig implements GetFunctionCodeSigningConfig; code
+// signing is unsupported, so it only proves the function exists.
+func (d Deps) getFunctionCodeSigningConfig(ctx context.Context, c *call) (reply, error) {
+	if _, _, err := d.lookup(ctx, c, ""); err != nil {
+		return reply{}, err
+	}
+	return reply{Status: http.StatusOK, Body: noCodeSigning}, nil
+}
+
+// listVersionsByFunction implements ListVersionsByFunction while $LATEST is
+// the only version. Paging is refused rather than guessed at.
+func (d Deps) listVersionsByFunction(ctx context.Context, c *call) (reply, error) {
+	if raw, ok := c.Query["MaxItems"]; ok {
+		n, err := strconv.Atoi(raw[0])
+		if err != nil || n < 1 || n > maxListItems {
+			return reply{}, invalid("MaxItems must be between 1 and %d", maxListItems)
+		}
+	}
+	if c.Query.Has("Marker") {
+		return reply{}, unsupported("Marker")
+	}
+	rec, cfg, err := d.loadConfiguration(ctx, c, "")
+	if err != nil {
+		return reply{}, err
+	}
+	cfg.FunctionArn = aws.String(arn.FormatLambdaFunction(rec.Region, rec.AccountID, rec.FunctionName) + ":$LATEST")
+	return reply{Status: http.StatusOK, Body: &lambda.ListVersionsByFunctionOutput{Versions: []*lambda.FunctionConfiguration{cfg}}}, nil
+}
+
+// getFunction implements GetFunction for the $LATEST version.
+func (d Deps) getFunction(ctx context.Context, c *call) (reply, error) {
+	rec, cfg, err := d.loadConfiguration(ctx, c, c.Query.Get("Qualifier"))
+	if err != nil {
+		return reply{}, err
 	}
 	out := &lambda.GetFunctionOutput{
 		Configuration: cfg,
@@ -11,6 +11,7 @@ import (
 	"errors"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"runtime"
 	"strings"
 	"sync"
@@ -662,8 +663,316 @@ func TestAuthorization(t *testing.T) {

 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateFunction", "GetFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "ListVersionsByFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
+	assert.Equal(t, "GetFunctionCodeSigningConfig", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2020-06-30/functions/x/code-signing-config", nil)))
 	assert.Equal(t, "CreateFunction", reg.ResolveAction(httptest.NewRequest(http.MethodPost, "/2015-03-31/functions", nil)))
 	assert.Equal(t, "GetFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x", nil)))
+	assert.Equal(t, "GetFunctionConfiguration", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/configuration", nil)))
 	assert.Empty(t, reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x", nil)))
 }
+
+func TestGetFunctionConfiguration_ReturnsTheSameConfigurationAsGetFunction(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+
+	rec, err := e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+	require.NoError(t, err)
+	var full lambda.GetFunctionOutput
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &full))
+
+	rec, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, rec.Code)
+	assert.NotContains(t, rec.Body.String(), `"Code"`, "the configuration is the top-level body, not GetFunction's wrapper")
+	var cfg lambda.FunctionConfiguration
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &cfg))
+	assert.Equal(t, *full.Configuration, cfg)
+	assert.Equal(t, "prod", *cfg.Environment.Variables["MODE"])
+
+	arn := "arn%3Aaws%3Alambda%3A" + gwtest.Region + "%3A" + gwtest.AccountID + "%3Afunction%3Ahello"
+	for _, path := range []string{
+		"/2015-03-31/functions/hello/configuration?Qualifier=%24LATEST",
+		"/2015-03-31/functions/" + arn + "/configuration",
+		"/2015-03-31/functions/" + arn + "%3A%24LATEST/configuration",
+	} {
+		rec, err := e.do(http.MethodGet, path, nil)
+		require.NoError(t, err, path)
+		assert.Equal(t, http.StatusOK, rec.Code, path)
+	}
+}
+
+func TestGetFunctionConfiguration_MissingAndUnsupportedMatchGetFunction(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	for label, tc := range map[string]struct{ suffix, code string }{
+		"missing":         {"ghost", awserrors.ErrorResourceNotFoundException},
+		"version":         {"hello?Qualifier=3", awserrors.ErrorResourceNotFoundException},
+		"other account":   {"arn%3Aaws%3Alambda%3A" + gwtest.Region + "%3A999999999999%3Afunction%3Ahello", awserrors.ErrorResourceNotFoundException},
+		"qualifier clash": {"hello%3A%24LATEST?Qualifier=3", awserrors.ErrorInvalidParameterValueException},
+		"not lambda":      {"arn%3Aaws%3As3%3A%3A%3Abucket", awserrors.ErrorInvalidParameterValueException},
+	} {
+		t.Run(label, func(t *testing.T) {
+			path, query, _ := strings.Cut(tc.suffix, "?")
+			if query != "" {
+				query = "?" + query
+			}
+			_, getErr := e.do(http.MethodGet, "/2015-03-31/functions/"+path+query, nil)
+			requireCode(t, getErr, tc.code)
+			_, err := e.do(http.MethodGet, "/2015-03-31/functions/"+path+"/configuration"+query, nil)
+			requireCode(t, err, tc.code)
+			assert.Equal(t, getErr.Error(), err.Error())
+		})
+	}
+
+	e.h.Region = "us-west-2"
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestGetFunctionConfiguration_DoesNotNeedTheCodeStore(t *testing.T) {
+	for name, codes := range map[string]awsapi.CodeStoreFunc{
+		"store outage": func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") },
+		"no store":     nil,
+	} {
+		t.Run(name, func(t *testing.T) {
+			e := newEnv(t)
+			_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+			require.NoError(t, err)
+			e.deps.Codes = codes
+
+			rec, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+			require.NoError(t, err)
+			require.Equal(t, http.StatusOK, rec.Code)
+			var cfg lambda.FunctionConfiguration
+			require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &cfg))
+			assert.Equal(t, "hello", *cfg.FunctionName)
+
+			_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+			requireCode(t, err, awserrors.ErrorServiceException)
+		})
+	}
+}
+
+func TestGetFunctionConfiguration_Authorization(t *testing.T) {
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	setup := func(t *testing.T, statements ...iamdomain.Statement) *env {
+		e := newEnv(t, statements...)
+		_, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+			FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+			Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+		})
+		require.NoError(t, err)
+		return e
+	}
+	get := func(e *env) error {
+		_, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+		return err
+	}
+
+	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))))
+	})
+	t.Run("a deny on the function ARN applies", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Deny", "lambda:GetFunctionConfiguration", arn))))
+	})
+	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionConfiguration", arn+"-other"))))
+	})
+	t.Run("an allow on the function ARN passes", func(t *testing.T) {
+		require.NoError(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionConfiguration", arn))))
+	})
+}
+
+func TestGetFunctionCodeSigningConfig_ExistingFunctionHasExplicitNulls(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	arn := "arn%3Aaws%3Alambda%3A" + gwtest.Region + "%3A" + gwtest.AccountID + "%3Afunction%3Ahello"
+	for _, name := range []string{"hello", arn, gwtest.AccountID + "%3Afunction%3Ahello"} {
+		rec, err := e.do(http.MethodGet, "/2020-06-30/functions/"+name+"/code-signing-config", nil)
+		require.NoError(t, err, name)
+		require.Equal(t, http.StatusOK, rec.Code, name)
+		assert.Equal(t, "application/json", rec.Header().Get("Content-Type"))
+		assert.JSONEq(t, `{"CodeSigningConfigArn":null,"FunctionName":null}`, rec.Body.String(), name)
+	}
+}
+
+func TestGetFunctionCodeSigningConfig_MissingFunctionIsNotFound(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	_, err = e.do(http.MethodGet, "/2020-06-30/functions/ghost/code-signing-config", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, getErr := e.do(http.MethodGet, "/2015-03-31/functions/ghost", nil)
+	assert.Equal(t, getErr.Error(), err.Error())
+
+	e.h.Region = "us-west-2"
+	_, err = e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestGetFunctionCodeSigningConfig_NeedsNeitherCodeStoreNorDecryption(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+	e.deps.Codes = func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") }
+	e.deps.AEAD = nil
+
+	rec, err := e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{"CodeSigningConfigArn":null,"FunctionName":null}`, rec.Body.String())
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	requireCode(t, err, awserrors.ErrorServiceException)
+
+	e.deps.Codes = nil
+	_, err = e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+	require.NoError(t, err)
+}
+
+func TestGetFunctionCodeSigningConfig_Authorization(t *testing.T) {
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	setup := func(t *testing.T, statements ...iamdomain.Statement) *env {
+		e := newEnv(t, statements...)
+		_, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+			FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+			Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+		})
+		require.NoError(t, err)
+		return e
+	}
+	get := func(e *env) error {
+		_, err := e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+		return err
+	}
+
+	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))))
+	})
+	t.Run("a deny on the function ARN applies", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Deny", "lambda:GetFunctionCodeSigningConfig", arn))))
+	})
+	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionCodeSigningConfig", arn+"-other"))))
+	})
+	t.Run("an allow on the function ARN passes", func(t *testing.T) {
+		require.NoError(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionCodeSigningConfig", arn))))
+	})
+}
+
+func TestListVersionsByFunction_ListsOnlyLatest(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+	rec, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	require.NoError(t, err)
+	var cfg lambda.FunctionConfiguration
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &cfg))
+
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	for _, path := range []string{
+		"/2015-03-31/functions/hello/versions",
+		"/2015-03-31/functions/hello/versions?MaxItems=10000",
+		"/2015-03-31/functions/hello/versions?MaxItems=1",
+		"/2015-03-31/functions/" + url.PathEscape(arn) + "/versions",
+	} {
+		rec, err := e.do(http.MethodGet, path, nil)
+		require.NoError(t, err, path)
+		require.Equal(t, http.StatusOK, rec.Code, path)
+		assert.NotContains(t, rec.Body.String(), "NextMarker", path)
+		var out lambda.ListVersionsByFunctionOutput
+		require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out), path)
+		require.Len(t, out.Versions, 1, path)
+		v := out.Versions[0]
+		assert.Equal(t, arn+":$LATEST", *v.FunctionArn, path)
+		assert.Equal(t, "$LATEST", *v.Version, path)
+		assert.Equal(t, "prod", *v.Environment.Variables["MODE"], path)
+		v.FunctionArn = cfg.FunctionArn
+		assert.Equal(t, cfg, *v, "apart from the qualified ARN it is the configuration read")
+	}
+}
+
+func TestListVersionsByFunction_RefusesBadMaxItemsAndAnyMarker(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	for _, q := range []string{"MaxItems=0", "MaxItems=10001", "MaxItems=-1", "MaxItems=ten", "MaxItems=", "Marker=not-a-marker", "Marker="} {
+		_, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/versions?"+q, nil)
+		requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+}
+
+func TestListVersionsByFunction_MissingFunctionIsNotFound(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/ghost/versions", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, getErr := e.do(http.MethodGet, "/2015-03-31/functions/ghost", nil)
+	assert.Equal(t, getErr.Error(), err.Error())
+
+	e.h.Region = "us-west-2"
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/versions", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestListVersionsByFunction_DoesNotNeedTheCodeStore(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+	e.deps.Codes = func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") }
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/versions", nil)
+	require.NoError(t, err)
+	e.deps.Codes = nil
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/versions", nil)
+	require.NoError(t, err)
+}
+
+func TestListVersionsByFunction_Authorization(t *testing.T) {
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	setup := func(t *testing.T, statements ...iamdomain.Statement) *env {
+		e := newEnv(t, statements...)
+		_, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+			FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+			Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+		})
+		require.NoError(t, err)
+		return e
+	}
+	list := func(e *env) error {
+		_, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/versions", nil)
+		return err
+	}
+
+	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
+		gwtest.AssertDenied(t, list(setup(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))))
+	})
+	t.Run("a deny on the function ARN applies", func(t *testing.T) {
+		gwtest.AssertDenied(t, list(setup(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Deny", "lambda:ListVersionsByFunction", arn))))
+	})
+	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
+		gwtest.AssertDenied(t, list(setup(t, gwtest.Statement("Allow", "lambda:ListVersionsByFunction", arn+"-other"))))
+	})
+	t.Run("an allow on the function ARN passes", func(t *testing.T) {
+		require.NoError(t, list(setup(t, gwtest.Statement("Allow", "lambda:ListVersionsByFunction", arn))))
+	})
+}
@@ -117,7 +117,7 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		if decodeErr == nil {
 			resource = functionARN(inv.Region, inv.AccountID, aws.StringValue(c.Create.FunctionName))
 		}
-	case "GetFunction":
+	case "GetFunction", "GetFunctionConfiguration", "GetFunctionCodeSigningConfig", "ListVersionsByFunction":
 		if name, _, err := resolveFunctionName(inv.Region, inv.AccountID, params[0]); err == nil {
 			resource = functionARN(inv.Region, inv.AccountID, name)
 		}
@@ -133,7 +133,10 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 			return err
 		}
 	}
-	if d.Meta == nil || d.Codes == nil {
+	// Only GetFunction presigns code; the other reads are metadata only.
+	needsCodes := action != "GetFunctionConfiguration" && action != "GetFunctionCodeSigningConfig" &&
+		action != "ListVersionsByFunction"
+	if d.Meta == nil || (d.Codes == nil && needsCodes) {
 		return errors.New(awserrors.ErrorServiceException)
 	}

@@ -180,7 +183,13 @@ func (d Deps) checkPassRole(inv dispatch.Invocation, roleARN string) error {
 }

 func writeJSON(w http.ResponseWriter, out reply) {
-	body, err := jsonutil.BuildJSON(out.Body)
+	var body []byte
+	var err error
+	if raw, ok := out.Body.(rawJSON); ok {
+		body = raw
+	} else {
+		body, err = jsonutil.BuildJSON(out.Body)
+	}
 	if err != nil {
 		slog.Error("Failed to marshal Lambda response", "err", err)
 		w.Header().Set("Content-Type", "application/json")
@@ -21,6 +21,10 @@ type call struct {
 	Create *lambda.CreateFunctionInput
 }

+// rawJSON is a response body written as is, for a shape the SDK encoder
+// cannot produce.
+type rawJSON []byte
+
 // reply is a handler's success: the HTTP status and the SDK shape to encode.
 type reply struct {
 	Status int
@@ -41,6 +45,18 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.getFunction(ctx, c)
 		}},
+	{Method: "GET", Pattern: "/2015-03-31/functions/{FunctionName}/configuration", Action: "GetFunctionConfiguration",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.getFunctionConfiguration(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/functions/{FunctionName}/versions", Action: "ListVersionsByFunction",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.listVersionsByFunction(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2020-06-30/functions/{FunctionName}/code-signing-config", Action: "GetFunctionCodeSigningConfig",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.getFunctionCodeSigningConfig(ctx, c)
+		}},
 }

 // actionNames returns the distinct actions in routes in stable order.
@@ -2,10 +2,14 @@ package floating

 import (
 	"context"
+	"crypto/rand"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"log/slog"
 	"net/netip"
+	"slices"
+	"sort"
 	"strings"

 	"github.com/mulgadc/spinifex/spinifex/domains/network/external"
@@ -28,6 +32,9 @@ type Binding struct {
 	// GatewayVPCID names the VPC whose NAT gateway answers on the address. Such
 	// an address has no ENI, so this is what says which node must hold it.
 	GatewayVPCID string `json:"gateway_vpc_id,omitempty"`
+	// Gen is a token drawn at allocation, never reused, so a holder pass that
+	// read the binding before a release and re-allocation can tell it is stale.
+	Gen string `json:"gen,omitempty"`
 	// Holder is the provider host this node last confirmed the address on. It
 	// is observed state, written after the provider agreed, never identity.
 	Holder string `json:"holder,omitempty"`
@@ -53,6 +60,9 @@ type Config struct {
 	Pool external.ExternalPoolConfig
 	// HostID is this node's provider host ID, which the holder pass attaches to.
 	HostID string
+	// KnownHostIDs are the other nodes' provider host IDs. A holder the provider reports
+	// is logged only when it is one of these or HostID.
+	KnownHostIDs []string
 	// LocalPorts lists the OVS ports plugged on this node, keyed by port name.
 	LocalPorts func(ctx context.Context) (map[string]struct{}, error)
 	// LocalGateway reports whether vpcID's gateway chassis is this node.
@@ -64,6 +74,8 @@ type PoolAllocator struct {
 	provider Provider
 	store    Store
 	cfg      Config
+	// fixed is set when the provider hands out a pre-bought set of addresses.
+	fixed bool
 	// suspects holds unbound labelled addresses one reconcile pass has seen,
 	// so the next can collect them. See Reconcile.
 	suspects map[string]struct{}
@@ -75,6 +87,16 @@ var (
 	_ external.AddressClaimer = (*PoolAllocator)(nil)
 )

+// holderLabel names a provider-reported holder for a log or a stall reason. The value is
+// shown only if it is a host we configured; anything else, which the provider alone
+// vouches for, is not echoed.
+func (a *PoolAllocator) holderLabel(id string) string {
+	if id == "" || id == a.cfg.HostID || slices.Contains(a.cfg.KnownHostIDs, id) {
+		return id
+	}
+	return "unrecognised holder"
+}
+
 // New constructs an allocator bound to a single pool.
 func New(provider Provider, store Store, cfg Config) (*PoolAllocator, error) {
 	if provider == nil {
@@ -86,10 +108,20 @@ func New(provider Provider, store Store, cfg Config) (*PoolAllocator, error) {
 	if cfg.HostID == "" {
 		return nil, fmt.Errorf("floating: pool %q missing provider host id", cfg.Pool.Name)
 	}
-	if cfg.Pool.LabelPrefix == "" {
+	_, fixed := provider.(FixedSet)
+	if cfg.Pool.LabelPrefix == "" && !fixed {
 		return nil, fmt.Errorf("floating: pool %q missing label prefix", cfg.Pool.Name)
 	}
-	return &PoolAllocator{provider: provider, store: store, cfg: cfg, suspects: map[string]struct{}{}}, nil
+	return &PoolAllocator{provider: provider, store: store, cfg: cfg, fixed: fixed, suspects: map[string]struct{}{}}, nil
+}
+
+// newGen returns a fresh random token for a binding's Gen.
+func newGen() (string, error) {
+	var b [16]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", fmt.Errorf("floating: draw allocation token: %w", err)
+	}
+	return hex.EncodeToString(b[:]), nil
 }

 // Pool returns the pool this allocator manages.
@@ -105,6 +137,15 @@ func (a *PoolAllocator) Allocate(ctx context.Context, req external.AllocateReque
 		return netip.Addr{}, fmt.Errorf("floating: pool mismatch (got %q, bound to %q)", req.PoolName, a.cfg.Pool.Name)
 	}

+	if fs, ok := a.provider.(FixedSet); ok {
+		return a.allocateFixed(ctx, fs, req)
+	}
+
+	// Drawn before the create, so a failure here leaves no address behind.
+	gen, err := newGen()
+	if err != nil {
+		return netip.Addr{}, err
+	}
 	addr, err := a.provider.Create(ctx, a.label(req))
 	if err != nil {
 		return netip.Addr{}, capacityErr(req.Purpose, fmt.Errorf("floating: create address: %w", err), err)
@@ -121,6 +162,7 @@ func (a *PoolAllocator) Allocate(ctx context.Context, req external.AllocateReque
 		AllocationID: req.AllocationID,
 		ENIID:        req.ENIID,
 		InstanceID:   req.InstanceID,
+		Gen:          gen,
 	}
 	err = a.store.Mutate(ctx, a.cfg.Pool.Name, func(rec *Record) (bool, error) {
 		rec.Bindings[key] = binding
@@ -137,6 +179,53 @@ func (a *PoolAllocator) Allocate(ctx context.Context, req external.AllocateReque
 	return addr.IP, nil
 }

+// allocateFixed hands out a member of the provider's set that has no binding.
+// The member is chosen inside the record update, which reruns on a conflict,
+// so the choice is made against the record that is actually written.
+func (a *PoolAllocator) allocateFixed(ctx context.Context, fs FixedSet, req external.AllocateRequest) (netip.Addr, error) {
+	members, err := fs.Members(ctx)
+	if err != nil {
+		return netip.Addr{}, capacityErr(req.Purpose, fmt.Errorf("floating: list pool members: %w", err), err)
+	}
+	sort.Slice(members, func(i, j int) bool { return members[i].IP.Less(members[j].IP) })
+
+	gen, err := newGen()
+	if err != nil {
+		return netip.Addr{}, err
+	}
+	var picked Address
+	err = a.store.Mutate(ctx, a.cfg.Pool.Name, func(rec *Record) (bool, error) {
+		picked = Address{}
+		for _, m := range members {
+			if !m.IP.IsValid() {
+				continue
+			}
+			key := m.IP.String()
+			if _, used := rec.Bindings[key]; used {
+				continue
+			}
+			rec.Bindings[key] = Binding{
+				AddrID:       m.ID,
+				Purpose:      req.Purpose,
+				AllocationID: req.AllocationID,
+				ENIID:        req.ENIID,
+				InstanceID:   req.InstanceID,
+				Gen:          gen,
+			}
+			picked = m
+			return true, nil
+		}
+		return false, fmt.Errorf("floating: pool %s has no free address: %w", a.cfg.Pool.Name, ErrQuotaExceeded)
+	})
+	if err != nil {
+		return netip.Addr{}, capacityErr(req.Purpose, fmt.Errorf("floating: allocate from pool: %w", err), err)
+	}
+	slog.InfoContext(ctx, "floating allocated external IP",
+		"pool", a.cfg.Pool.Name, "public_ip", picked.IP.String(), "addr_id", picked.ID,
+		"purpose", req.Purpose, "eni_id", req.ENIID)
+	return picked.IP, nil
+}
+
 // Release drops the binding and deletes the address. ownerENIID scopes the
 // release as StaticPoolAllocator.Release does, so a teardown sweep naming a
 // stale owner is a no-op rather than a double-free.
@@ -242,7 +331,11 @@ func (a *PoolAllocator) undo(ctx context.Context, addr Address) {
 // destroy detaches and then deletes an address; a missing one is already gone.
 // Vultr documents delete as detaching implicitly, but has been measured to
 // refuse it with "Unable to detach subnet" while the address is attached.
+// A fixed set's members belong to whoever bought them, so nothing is destroyed.
 func (a *PoolAllocator) destroy(ctx context.Context, addrID string) error {
+	if a.fixed {
+		return nil
+	}
 	if err := a.provider.Detach(ctx, addrID); err != nil && !errors.Is(err, ErrNotFound) {
 		return fmt.Errorf("detach: %w", err)
 	}
@@ -0,0 +1,401 @@
+package floating
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"maps"
+	"net/netip"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/network/external"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// fixedFake is a provider whose addresses exist before the pool does: it cannot create, and
+// detach and delete change nothing.
+type fixedFake struct {
+	mu    sync.Mutex
+	addrs map[string]*Address
+	calls []string
+	// onList runs once when the holder pass lists, which is after it read the bindings and before
+	// it acts on them.
+	onList func()
+}
+
+var (
+	_ Provider = (*fixedFake)(nil)
+	_ FixedSet = (*fixedFake)(nil)
+)
+
+func newFixedFake(n int) *fixedFake {
+	f := &fixedFake{addrs: map[string]*Address{}}
+	for i := 1; i <= n; i++ {
+		id := fmt.Sprintf("eip-%d", i)
+		f.addrs[id] = &Address{ID: id, IP: netip.MustParseAddr(fmt.Sprintf("198.51.100.%d", i))}
+	}
+	return f
+}
+
+func (f *fixedFake) record(call string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.calls = append(f.calls, call)
+}
+
+func (f *fixedFake) callsWith(prefix string) int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	n := 0
+	for _, c := range f.calls {
+		if len(c) >= len(prefix) && c[:len(prefix)] == prefix {
+			n++
+		}
+	}
+	return n
+}
+
+func (f *fixedFake) holder(id string) string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.addrs[id].HolderID
+}
+
+func (f *fixedFake) snapshot() []Address {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := make([]Address, 0, len(f.addrs))
+	for _, a := range f.addrs {
+		out = append(out, *a)
+	}
+	return out
+}
+
+func (f *fixedFake) Create(context.Context, string) (Address, error) {
+	f.record("create")
+	return Address{}, errors.New("a fixed set creates nothing")
+}
+
+func (f *fixedFake) Attach(_ context.Context, id, host string) error {
+	f.record("attach:" + id + ">" + host)
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.addrs[id].HolderID = host
+	return nil
+}
+
+func (f *fixedFake) Detach(context.Context, string) error { f.record("detach"); return nil }
+func (f *fixedFake) Delete(context.Context, string) error { f.record("delete"); return nil }
+
+func (f *fixedFake) Get(_ context.Context, id string) (Address, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	a, ok := f.addrs[id]
+	if !ok {
+		return Address{}, ErrNotFound
+	}
+	return *a, nil
+}
+
+func (f *fixedFake) List(context.Context) ([]Address, error) {
+	f.mu.Lock()
+	hook := f.onList
+	f.onList = nil
+	f.mu.Unlock()
+	if hook != nil {
+		hook()
+	}
+	return f.snapshot(), nil
+}
+
+func (f *fixedFake) Members(context.Context) ([]Address, error) { return f.snapshot(), nil }
+
+func newFixedNode(t *testing.T, fake *fixedFake, store Store, hostID string) *node {
+	t.Helper()
+	n := &node{plugged: map[string]struct{}{}, gateway: map[string]bool{}}
+	alloc, err := New(fake, store, Config{
+		Pool:   external.ExternalPoolConfig{Name: pool, Source: "latitude"},
+		HostID: hostID,
+		LocalPorts: func(context.Context) (map[string]struct{}, error) {
+			return maps.Clone(n.plugged), nil
+		},
+		LocalGateway: func(_ context.Context, vpcID string) (bool, error) { return n.gateway[vpcID], nil },
+	})
+	require.NoError(t, err)
+	n.alloc = alloc
+	return n
+}
+
+func fixedSetup(t *testing.T, members int) (*fixedFake, *memStore, *node, *node) {
+	t.Helper()
+	fake := newFixedFake(members)
+	store := newMemStore()
+	return fake, store, newFixedNode(t, fake, store, hostA), newFixedNode(t, fake, store, hostB)
+}
+
+func TestFixedSet_AllocatePicksAFreeMemberAndCreatesNothing(t *testing.T) {
+	fake, store, a, _ := fixedSetup(t, 2)
+	ip1 := allocENI(t, a, "eni-1")
+	ip2 := allocENI(t, a, "eni-2")
+
+	assert.NotEqual(t, ip1, ip2)
+	assert.Equal(t, 0, fake.callsWith("create"))
+	b := store.binding(t, ip1)
+	assert.NotEmpty(t, b.Gen)
+	assert.NotEqual(t, b.Gen, store.binding(t, ip2).Gen)
+	assert.Empty(t, fake.holder(b.AddrID), "the owner's node attaches, never the allocating node")
+}
+
+func TestFixedSet_NoFreeMemberIsQuotaForTheCall(t *testing.T) {
+	_, _, a, _ := fixedSetup(t, 1)
+	allocENI(t, a, "eni-1")
+
+	_, err := a.alloc.Allocate(context.Background(), external.AllocateRequest{PoolName: pool, Purpose: purposeEIP})
+	assert.ErrorContains(t, err, awserrors.ErrorAddressLimitExceeded)
+	_, err = a.alloc.Allocate(context.Background(), external.AllocateRequest{PoolName: pool, Purpose: "eni-public"})
+	assert.ErrorContains(t, err, awserrors.ErrorInsufficientAddressCapacity)
+}
+
+func TestFixedSet_ConcurrentAllocationsNeverShareAMember(t *testing.T) {
+	fake := newFixedFake(3)
+	store := newMemStore()
+	const callers = 8
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	got := map[netip.Addr]int{}
+	failed := 0
+	for i := range callers {
+		n := newFixedNode(t, fake, store, hostA)
+		wg.Go(func() {
+			ip, err := n.alloc.Allocate(context.Background(), external.AllocateRequest{
+				PoolName: pool, Purpose: "eni-public", ENIID: fmt.Sprintf("eni-%d", i),
+			})
+			mu.Lock()
+			defer mu.Unlock()
+			if err != nil {
+				failed++
+				return
+			}
+			got[ip]++
+		})
+	}
+	wg.Wait()
+
+	assert.Len(t, got, 3)
+	for ip, n := range got {
+		assert.Equal(t, 1, n, "%s was handed out twice", ip)
+	}
+	assert.Equal(t, callers-3, failed)
+}
+
+func TestFixedSet_ReleaseFreesTheMemberAndTouchesNothingAtTheProvider(t *testing.T) {
+	fake, store, a, _ := fixedSetup(t, 1)
+	ip := allocENI(t, a, "eni-1")
+	first := store.binding(t, ip).Gen
+
+	require.NoError(t, a.alloc.Release(context.Background(), pool, ip, "eni-1"))
+	assert.Len(t, fake.snapshot(), 1, "a member is never deleted")
+
+	again := allocENI(t, a, "eni-2")
+	assert.Equal(t, ip, again, "the freed member is handed out again")
+	assert.NotEqual(t, first, store.binding(t, again).Gen, "a reused address is a new allocation")
+}
+
+func TestFixedSet_ClaimMovesTheMemberToTheOwnersNode(t *testing.T) {
+	fake, store, a, b := fixedSetup(t, 1)
+	ip := allocENI(t, a, "eni-1")
+	a.plug("eni-1")
+	_, err := a.alloc.ClaimLocalAddresses(context.Background())
+	require.NoError(t, err)
+	addrID := store.binding(t, ip).AddrID
+	require.Equal(t, hostA, fake.holder(addrID))
+
+	a.plugged = map[string]struct{}{}
+	b.plug("eni-1")
+	res, err := b.alloc.ClaimLocalAddresses(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, []string{ip.String()}, res.Claimed)
+	assert.Equal(t, hostB, fake.holder(addrID))
+	assert.Equal(t, hostB, store.binding(t, ip).Holder)
+}
+
+func TestFixedSet_AnUnownedMemberIsNotSurrendered(t *testing.T) {
+	fake, store, a, _ := fixedSetup(t, 1)
+	ip, err := a.alloc.Allocate(context.Background(), external.AllocateRequest{PoolName: pool, Purpose: purposeEIP, AllocationID: "eipalloc-1"})
+	require.NoError(t, err)
+	addrID := store.binding(t, ip).AddrID
+	require.NoError(t, fake.Attach(context.Background(), addrID, hostA))
+	fake.calls = nil
+
+	_, err = a.alloc.ClaimLocalAddresses(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 0, fake.callsWith("detach"), "detach changes nothing at the provider, so the pass must not pretend it did")
+	assert.Equal(t, hostA, fake.holder(addrID))
+}
+
+func TestFixedSet_ReconcileNeverCollectsAMember(t *testing.T) {
+	fake, _, a, _ := fixedSetup(t, 2)
+	for range 2 {
+		res, err := a.alloc.Reconcile(context.Background())
+		require.NoError(t, err)
+		assert.Empty(t, res.Collected)
+		assert.Equal(t, 2, res.Skipped)
+	}
+	assert.Equal(t, 0, fake.callsWith("delete"))
+}
+
+// A holder pass reads the bindings, then acts. Releasing and re-allocating in between puts the
+// same address and provider ID under a different owner, which the pass must not move.
+func TestFixedSet_PassPausedAcrossReallocationDoesNotMove(t *testing.T) {
+	fake, store, a, b := fixedSetup(t, 1)
+	ip := allocENI(t, a, "eni-1")
+	b.plug("eni-1")
+	fake.onList = func() {
+		require.NoError(t, a.alloc.Release(context.Background(), pool, ip, "eni-1"))
+		again := allocENI(t, a, "eni-2")
+		require.Equal(t, ip, again)
+	}
+
+	res, err := b.alloc.ClaimLocalAddresses(context.Background())
+	require.NoError(t, err)
+	assert.Empty(t, res.Claimed)
+	assert.Empty(t, res.Stalled)
+	assert.Equal(t, 0, fake.callsWith("attach"), "the stale pass must not move the new allocation")
+	assert.Equal(t, "eni-2", store.binding(t, ip).ENIID)
+	assert.Empty(t, store.binding(t, ip).Holder)
+}
+
+func TestFixedSet_PassPausedAcrossAGatewayVPCChangeDoesNotMove(t *testing.T) {
+	fake, store, a, b := fixedSetup(t, 1)
+	ip, err := a.alloc.Allocate(context.Background(), external.AllocateRequest{PoolName: pool, Purpose: purposeEIP, AllocationID: "eipalloc-nat"})
+	require.NoError(t, err)
+	binder := NewGatewayBinder(store, []string{pool})
+	require.NoError(t, binder.BindGateway(context.Background(), ip.String(), "vpc-1"))
+	b.gateway["vpc-1"] = true
+	fake.onList = func() {
+		require.NoError(t, binder.BindGateway(context.Background(), ip.String(), "vpc-2"))
+	}
+
+	res, err := b.alloc.ClaimLocalAddresses(context.Background())
+	require.NoError(t, err)
+	assert.Empty(t, res.Claimed)
+	assert.Equal(t, 0, fake.callsWith("attach"))
+	assert.Equal(t, "vpc-2", store.binding(t, ip).GatewayVPCID)
+}
+
+func TestRecordHolder_IgnoresAStaleAllocation(t *testing.T) {
+	_, store, a, _ := fixedSetup(t, 1)
+	ip := allocENI(t, a, "eni-1")
+	live := store.binding(t, ip)
+
+	stale := live
+	stale.Gen = "an-earlier-allocation"
+	a.alloc.recordHolder(context.Background(), ip.String(), stale, hostA)
+	assert.Empty(t, store.binding(t, ip).Holder)
+
+	a.alloc.recordHolder(context.Background(), ip.String(), live, hostA)
+	assert.Equal(t, hostA, store.binding(t, ip).Holder)
+}
+
+func TestNew_AFixedSetNeedsNoLabelPrefix(t *testing.T) {
+	_, err := New(newFixedFake(1), newMemStore(), Config{Pool: external.ExternalPoolConfig{Name: pool}, HostID: hostA})
+	require.NoError(t, err)
+}
+
+func TestAllocate_EachAllocationGetsItsOwnGen(t *testing.T) {
+	_, store, a, _ := setup(t)
+	ip1 := allocENI(t, a, "eni-1")
+	ip2 := allocENI(t, a, "eni-2")
+	assert.NotEmpty(t, store.binding(t, ip1).Gen)
+	assert.NotEqual(t, store.binding(t, ip1).Gen, store.binding(t, ip2).Gen)
+}
+
+// failingGets lets the first n reads through, then fails every later one.
+type failingGets struct {
+	Store
+
+	mu   sync.Mutex
+	left int
+}
+
+func (s *failingGets) Get(ctx context.Context, poolName string) (Record, error) {
+	s.mu.Lock()
+	ok := s.left > 0
+	s.left--
+	s.mu.Unlock()
+	if !ok {
+		return Record{}, errors.New("store unavailable")
+	}
+	return s.Store.Get(ctx, poolName)
+}
+
+// A pass that could not re-read the binding moved nothing, so it must say so: the daemon
+// keeps an address's unreachable warning only while it appears in Stalled.
+func TestFixedSet_FailedReReadIsStalledNotSuccess(t *testing.T) {
+	fake, store, a, _ := fixedSetup(t, 1)
+	ip := allocENI(t, a, "eni-1")
+	a.plug("eni-1")
+	_, err := a.alloc.ClaimLocalAddresses(context.Background())
+	require.NoError(t, err)
+	addrID := store.binding(t, ip).AddrID
+
+	flaky := &failingGets{Store: store, left: 1}
+	b := newFixedNode(t, fake, flaky, hostB)
+	b.plug("eni-1")
+	fake.calls = nil
+
+	res, err := b.alloc.ClaimLocalAddresses(context.Background())
+	require.NoError(t, err)
+	require.Len(t, res.Stalled, 1)
+	assert.Equal(t, ip.String(), res.Stalled[0].PublicIP)
+	assert.Contains(t, res.Stalled[0].Reason, "re-read")
+	assert.Empty(t, res.Claimed)
+	assert.Equal(t, 0, fake.callsWith("attach"))
+	assert.Equal(t, hostA, fake.holder(addrID), "nothing moved")
+}
+
+func TestClaim_FailedReReadIsStalledForVultrToo(t *testing.T) {
+	fake, store, a, _ := setup(t)
+	ip := allocENI(t, a, "eni-1")
+	a.plug("eni-1")
+	_, err := a.alloc.ClaimLocalAddresses(context.Background())
+	require.NoError(t, err)
+	flaky := &failingGets{Store: store, left: 1}
+	b := newNode(t, fake, flaky, hostB)
+	b.plug("eni-1")
+
+	res, err := b.alloc.ClaimLocalAddresses(context.Background())
+	require.NoError(t, err)
+	require.Len(t, res.Stalled, 1)
+	assert.Equal(t, ip.String(), res.Stalled[0].PublicIP)
+}
+
+func TestFixedSet_ReleaseNeverDetachesOrDeletes(t *testing.T) {
+	fake, _, a, _ := fixedSetup(t, 1)
+	ip := allocENI(t, a, "eni-1")
+	require.NoError(t, a.alloc.Release(context.Background(), pool, ip, "eni-1"))
+	assert.Equal(t, 0, fake.callsWith("detach"))
+	assert.Equal(t, 0, fake.callsWith("delete"))
+}
+
+func TestFixedSet_ReconcileNeverDetachesOrDeletesAndStillReportsMissing(t *testing.T) {
+	fake, store, a, _ := fixedSetup(t, 2)
+	// A member the provider no longer lists, bound in the record.
+	ip := allocENI(t, a, "eni-1")
+	delete(fake.addrs, store.binding(t, ip).AddrID)
+	// An unbound member that happens to carry a matching label.
+	fake.addrs["eip-2"].Label = prefix + ":stray"
+
+	for range 3 {
+		res, err := a.alloc.Reconcile(context.Background())
+		require.NoError(t, err)
+		assert.Empty(t, res.Collected)
+		assert.Equal(t, []string{ip.String()}, res.Missing)
+	}
+	assert.Equal(t, 0, fake.callsWith("detach"))
+	assert.Equal(t, 0, fake.callsWith("delete"))
+}
@@ -65,7 +65,7 @@ func (a *PoolAllocator) ClaimLocalAddresses(ctx context.Context) (external.Claim
 		case local:
 			res.Local++
 			a.claim(ctx, key, b, addr, &res)
-		case !b.owned() && addr.HolderID == a.cfg.HostID:
+		case !b.owned() && addr.HolderID == a.cfg.HostID && !a.fixed:
 			a.surrender(ctx, key, b)
 		}
 	}
@@ -78,17 +78,33 @@ func (a *PoolAllocator) ClaimLocalAddresses(ctx context.Context) (external.Claim
 func (a *PoolAllocator) claim(ctx context.Context, key string, b Binding, addr Address, res *external.ClaimResult) {
 	host := a.cfg.HostID
 	if addr.HolderID == host {
-		a.recordHolder(ctx, key, b.AddrID, host)
+		a.recordHolder(ctx, key, b, host)
 		return
 	}
 	stall := func(reason string) {
 		res.Stalled = append(res.Stalled, external.StalledAddress{PublicIP: key, ENIID: b.ENIID, Reason: reason})
 	}
+	// ok is false when the binding changed, which needs no report: the new
+	// allocation is handled on its own. A failed re-read is reported, since the
+	// address was not moved and the pass must not read as complete.
+	stillOurs := func() bool {
+		ok, err := a.unchanged(ctx, key, b)
+		if err != nil {
+			stall(fmt.Sprintf("re-read binding for %s before moving it: %v", b.AddrID, err))
+		}
+		return ok
+	}
 	if addr.HolderID != "" {
-		if err := a.provider.Detach(ctx, b.AddrID); err != nil && !errors.Is(err, ErrNotFound) {
-			stall(fmt.Sprintf("detach %s from %s: %v", b.AddrID, addr.HolderID, err))
+		if !stillOurs() {
 			return
 		}
+		if err := a.provider.Detach(ctx, b.AddrID); err != nil && !errors.Is(err, ErrNotFound) {
+			stall(fmt.Sprintf("detach %s from %s: %v", b.AddrID, a.holderLabel(addr.HolderID), err))
+			return
+		}
+	}
+	if !stillOurs() {
+		return
 	}
 	if err := a.provider.Attach(ctx, b.AddrID, host); err != nil {
 		stall(fmt.Sprintf("attach %s to %s: %v", b.AddrID, host, err))
@@ -100,7 +116,7 @@ func (a *PoolAllocator) claim(ctx context.Context, key string, b Binding, addr A
 		return
 	}
 	if got.HolderID != host {
-		stall(fmt.Sprintf("provider accepted the attach of %s to %s and reports holder %q", b.AddrID, host, got.HolderID))
+		stall(fmt.Sprintf("provider accepted the attach of %s to %s and reports holder %q", b.AddrID, host, a.holderLabel(got.HolderID)))
 		return
 	}
 	// A move from another host is worth an operator's attention; a first
@@ -111,14 +127,21 @@ func (a *PoolAllocator) claim(ctx context.Context, key string, b Binding, addr A
 	}
 	slog.Log(ctx, level, "floating claimed an address whose owner runs here",
 		"pool", a.cfg.Pool.Name, "public_ip", key, "addr_id", b.AddrID,
-		"eni_id", b.ENIID, "gateway_vpc_id", b.GatewayVPCID, "from_host", addr.HolderID, "to_host", host)
-	a.recordHolder(ctx, key, b.AddrID, host)
+		"eni_id", b.ENIID, "gateway_vpc_id", b.GatewayVPCID, "from_host", a.holderLabel(addr.HolderID), "to_host", host)
+	a.recordHolder(ctx, key, b, host)
 	res.Claimed = append(res.Claimed, key)
 }

 // surrender detaches an address this node holds with no owner, which is what a
 // disassociate leaves: still allocated and billing, answered by no guest.
 func (a *PoolAllocator) surrender(ctx context.Context, key string, b Binding) {
+	if ok, err := a.unchanged(ctx, key, b); !ok {
+		if err != nil {
+			slog.WarnContext(ctx, "floating could not re-read the binding before surrendering; retrying next pass",
+				"pool", a.cfg.Pool.Name, "public_ip", key, "error", err)
+		}
+		return
+	}
 	if err := a.provider.Detach(ctx, b.AddrID); err != nil && !errors.Is(err, ErrNotFound) {
 		slog.WarnContext(ctx, "floating could not detach an unowned address; retrying next pass",
 			"pool", a.cfg.Pool.Name, "public_ip", key, "addr_id", b.AddrID, "error", err)
@@ -126,7 +149,7 @@ func (a *PoolAllocator) surrender(ctx context.Context, key string, b Binding) {
 	}
 	slog.InfoContext(ctx, "floating detached an address with no owner",
 		"pool", a.cfg.Pool.Name, "public_ip", key, "addr_id", b.AddrID)
-	a.recordHolder(ctx, key, b.AddrID, "")
+	a.recordHolder(ctx, key, b, "")
 }

 // ownsBinding answers whether this node is where b's traffic is handled. An
@@ -146,13 +169,33 @@ func (a *PoolAllocator) ownsBinding(ctx context.Context, b Binding, plugged map[
 	return local, nil
 }

+// unchanged reads the binding again and reports whether it is still the one
+// this pass snapshotted: same allocation, owner and gateway. A release and
+// re-allocation reuses the address and its provider ID, so the allocation
+// token is what tells them apart. The re-read narrows the window before the
+// provider call but does not close it: a stale move is repaired only when the
+// owning node next runs its claim pass and the provider accepts the move.
+func (a *PoolAllocator) unchanged(ctx context.Context, key string, snap Binding) (bool, error) {
+	rec, err := a.store.Get(ctx, a.cfg.Pool.Name)
+	if err != nil {
+		return false, err
+	}
+	cur, ok := rec.Bindings[key]
+	if ok && cur.Gen == snap.Gen && cur.AddrID == snap.AddrID && cur.ENIID == snap.ENIID && cur.GatewayVPCID == snap.GatewayVPCID {
+		return true, nil
+	}
+	slog.InfoContext(ctx, "floating holder skipped an address whose binding changed during the pass",
+		"pool", a.cfg.Pool.Name, "public_ip", key, "addr_id", snap.AddrID)
+	return false, nil
+}
+
 // recordHolder writes the observed holder under CAS, only while the binding
-// still names the address this pass acted on. A failed write is logged: the
+// is still the allocation this pass acted on. A failed write is logged: the
 // provider already agrees, and the next pass rewrites it.
-func (a *PoolAllocator) recordHolder(ctx context.Context, key, addrID, holder string) {
+func (a *PoolAllocator) recordHolder(ctx context.Context, key string, snap Binding, holder string) {
 	err := a.store.Mutate(ctx, a.cfg.Pool.Name, func(r *Record) (bool, error) {
 		b, ok := r.Bindings[key]
-		if !ok || b.AddrID != addrID || b.Holder == holder {
+		if !ok || b.AddrID != snap.AddrID || b.Gen != snap.Gen || b.Holder == holder {
 			return false, nil
 		}
 		b.Holder = holder
@@ -0,0 +1,124 @@
+package floating
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/latitude"
+)
+
+// latitudeAPI is the slice of the latitude client the adapter calls, so tests can
+// stand the fake in for it.
+type latitudeAPI interface {
+	GetElasticIP(ctx context.Context, id string) (latitude.ElasticIP, error)
+	ListElasticIPs(ctx context.Context) ([]latitude.ElasticIP, error)
+	MoveElasticIP(ctx context.Context, id, serverID string) error
+}
+
+var (
+	_ latitudeAPI = (*latitude.Client)(nil)
+	_ latitudeAPI = (*latitude.Fake)(nil)
+)
+
+// LatitudeProvider adapts a fixed set of latitude.sh Elastic IPs to Provider.
+// The set is named by ID, since the API has no labels to tell ours from the
+// team's others. Addresses are never created, detached or deleted here: the API
+// has no unassign, and the addresses belong to whoever bought them.
+type LatitudeProvider struct {
+	api latitudeAPI
+	ids []string
+}
+
+var (
+	_ Provider = (*LatitudeProvider)(nil)
+	_ FixedSet = (*LatitudeProvider)(nil)
+)
+
+// NewLatitudeProvider wraps a latitude client or fake over the given Elastic IP IDs.
+func NewLatitudeProvider(api latitudeAPI, ids []string) (*LatitudeProvider, error) {
+	if len(ids) == 0 {
+		return nil, errors.New("floating: latitude pool names no Elastic IP IDs")
+	}
+	return &LatitudeProvider{api: api, ids: slices.Clone(ids)}, nil
+}
+
+func fromLatitude(e latitude.ElasticIP) Address {
+	return Address{ID: e.ID, IP: e.Address, HolderID: e.ServerID}
+}
+
+// mapLatitude wraps a latitude error around the engine's sentinel for it.
+func mapLatitude(err error) error {
+	switch {
+	case err == nil:
+		return nil
+	case errors.Is(err, latitude.ErrNotFound):
+		return fmt.Errorf("%w: %w", ErrNotFound, err)
+	}
+	return err
+}
+
+func (p *LatitudeProvider) member(id string) bool { return slices.Contains(p.ids, id) }
+
+// Create is refused: the set is fixed.
+func (p *LatitudeProvider) Create(context.Context, string) (Address, error) {
+	return Address{}, errors.New("floating: a latitude pool cannot create addresses")
+}
+
+// Attach moves the address to hostID and returns once it is active there.
+func (p *LatitudeProvider) Attach(ctx context.Context, addrID, hostID string) error {
+	if !p.member(addrID) {
+		return fmt.Errorf("%w: %s is not in this pool", ErrNotFound, addrID)
+	}
+	return mapLatitude(p.api.MoveElasticIP(ctx, addrID, hostID))
+}
+
+// Detach does nothing: latitude.sh cannot unassign an address, only move it.
+func (p *LatitudeProvider) Detach(context.Context, string) error { return nil }
+
+// Delete does nothing: the addresses are not ours to release.
+func (p *LatitudeProvider) Delete(context.Context, string) error { return nil }
+
+func (p *LatitudeProvider) Get(ctx context.Context, addrID string) (Address, error) {
+	if !p.member(addrID) {
+		return Address{}, fmt.Errorf("%w: %s is not in this pool", ErrNotFound, addrID)
+	}
+	e, err := p.api.GetElasticIP(ctx, addrID)
+	if err != nil {
+		return Address{}, mapLatitude(err)
+	}
+	return fromLatitude(e), nil
+}
+
+// List returns the pool's members. The API lists the whole project, so the
+// result is cut to the configured IDs.
+func (p *LatitudeProvider) List(ctx context.Context) ([]Address, error) {
+	return p.Members(ctx)
+}
+
+// Members returns the configured addresses with their holders. An address the
+// API does not list is an error, since the engine could otherwise hand out a
+// set smaller than the operator configured without saying so.
+func (p *LatitudeProvider) Members(ctx context.Context) ([]Address, error) {
+	all, err := p.api.ListElasticIPs(ctx)
+	if err != nil {
+		return nil, mapLatitude(err)
+	}
+	byID := make(map[string]latitude.ElasticIP, len(all))
+	for _, e := range all {
+		byID[e.ID] = e
+	}
+	out := make([]Address, 0, len(p.ids))
+	for _, id := range p.ids {
+		e, ok := byID[id]
+		if !ok {
+			return nil, fmt.Errorf("floating: configured Elastic IP %s is not in the project listing", id)
+		}
+		if why := e.Unusable(); why != "" {
+			return nil, fmt.Errorf("floating: configured Elastic IP %s is unusable: %s", id, why)
+		}
+		out = append(out, fromLatitude(e))
+	}
+	return out, nil
+}
@@ -0,0 +1,263 @@
+package floating
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/network/external"
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/latitude"
+)
+
+func latitudeFake() *latitude.Fake {
+	f := latitude.NewFake("sv_a", "sv_b")
+	f.Seed(latitude.ElasticIP{ID: "eip_1", Address: netip.MustParseAddr("203.0.113.1")})
+	f.Seed(latitude.ElasticIP{ID: "eip_2", Address: netip.MustParseAddr("203.0.113.2"), ServerID: "sv_a"})
+	f.Seed(latitude.ElasticIP{ID: "eip_other", Address: netip.MustParseAddr("203.0.113.9")})
+	return f
+}
+
+func TestLatitudeProvider_MembersAreOnlyTheConfiguredIDs(t *testing.T) {
+	p, err := NewLatitudeProvider(latitudeFake(), []string{"eip_1", "eip_2"})
+	require.NoError(t, err)
+	got, err := p.Members(context.Background())
+	require.NoError(t, err)
+	require.Len(t, got, 2)
+	assert.Equal(t, "eip_1", got[0].ID)
+	assert.Equal(t, "sv_a", got[1].HolderID)
+	assert.Empty(t, got[0].Label)
+}
+
+func TestLatitudeProvider_AMissingMemberIsAnError(t *testing.T) {
+	p, err := NewLatitudeProvider(latitudeFake(), []string{"eip_1", "eip_gone"})
+	require.NoError(t, err)
+	_, err = p.Members(context.Background())
+	assert.ErrorContains(t, err, "eip_gone")
+}
+
+func TestLatitudeProvider_AttachMovesAndDetachDeleteChangeNothing(t *testing.T) {
+	f := latitudeFake()
+	p, err := NewLatitudeProvider(f, []string{"eip_1", "eip_2"})
+	require.NoError(t, err)
+	ctx := context.Background()
+
+	require.NoError(t, p.Attach(ctx, "eip_2", "sv_b"))
+	assert.Equal(t, "sv_b", f.HolderOf("eip_2"))
+	require.NoError(t, p.Detach(ctx, "eip_2"))
+	require.NoError(t, p.Delete(ctx, "eip_2"))
+	assert.Equal(t, "sv_b", f.HolderOf("eip_2"))
+	got, err := p.Get(ctx, "eip_2")
+	require.NoError(t, err)
+	assert.Equal(t, "sv_b", got.HolderID)
+}
+
+func TestLatitudeProvider_RefusesWhatIsNotInTheSet(t *testing.T) {
+	f := latitudeFake()
+	p, err := NewLatitudeProvider(f, []string{"eip_1"})
+	require.NoError(t, err)
+	ctx := context.Background()
+
+	assert.ErrorIs(t, p.Attach(ctx, "eip_other", "sv_b"), ErrNotFound)
+	assert.Empty(t, f.HolderOf("eip_other"))
+	_, err = p.Get(ctx, "eip_other")
+	assert.ErrorIs(t, err, ErrNotFound)
+	_, err = p.Create(ctx, "x")
+	assert.Error(t, err)
+}
+
+func TestLatitudeProvider_MapsNotFoundAndKeepsRefusals(t *testing.T) {
+	f := latitudeFake()
+	p, err := NewLatitudeProvider(f, []string{"eip_1"})
+	require.NoError(t, err)
+	ctx := context.Background()
+
+	assert.ErrorIs(t, p.Attach(ctx, "eip_1", "sv_unknown"), latitude.ErrMoveRefused)
+	f.Fail["get"] = latitude.ErrNotFound
+	_, err = p.Get(ctx, "eip_1")
+	assert.ErrorIs(t, err, ErrNotFound)
+}
+
+func TestNewLatitudeProvider_NeedsIDs(t *testing.T) {
+	_, err := NewLatitudeProvider(latitudeFake(), nil)
+	assert.Error(t, err)
+}
+
+// The engine over the adapter: allocation picks, the owner's node claims, and
+// running out is the capacity error.
+func TestLatitudeProvider_EngineEndToEnd(t *testing.T) {
+	f := latitudeFake()
+	p, err := NewLatitudeProvider(f, []string{"eip_1", "eip_2"})
+	require.NoError(t, err)
+	store := newMemStore()
+	n := &node{plugged: map[string]struct{}{}, gateway: map[string]bool{}}
+	alloc, err := New(p, store, Config{
+		Pool:       external.ExternalPoolConfig{Name: pool, Source: "latitude"},
+		HostID:     "sv_b",
+		LocalPorts: func(context.Context) (map[string]struct{}, error) { return n.plugged, nil },
+	})
+	require.NoError(t, err)
+	n.alloc = alloc
+
+	ip := allocENI(t, n, "eni-1")
+	assert.Equal(t, netip.MustParseAddr("203.0.113.1"), ip)
+	n.plug("eni-1")
+	res, err := alloc.ClaimLocalAddresses(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, []string{ip.String()}, res.Claimed)
+	assert.Equal(t, "sv_b", f.HolderOf("eip_1"))
+
+	allocENI(t, n, "eni-2")
+	_, err = alloc.Allocate(context.Background(), external.AllocateRequest{PoolName: pool, Purpose: "eni-public", ENIID: "eni-3"})
+	assert.ErrorContains(t, err, "InsufficientAddressCapacity")
+}
+
+func TestProviderFor_PicksByPoolSource(t *testing.T) {
+	p, err := providerFor(external.ExternalPoolConfig{
+		Name: pool, Source: external.SourceLatitude, LatitudeProjectID: "proj_1", LatitudeElasticIPIDs: []string{"eip_1"},
+	})
+	require.NoError(t, err)
+	_, isFixed := p.(FixedSet)
+	assert.True(t, isFixed)
+
+	v, err := providerFor(external.ExternalPoolConfig{Name: pool, Source: external.SourceVultr, VultrRegion: "ord"})
+	require.NoError(t, err)
+	_, isFixed = v.(FixedSet)
+	assert.False(t, isFixed)
+
+	_, err = providerFor(external.ExternalPoolConfig{Name: pool, Source: external.SourceStatic})
+	assert.ErrorContains(t, err, "not a floating provider")
+	_, err = providerFor(external.ExternalPoolConfig{Name: pool, Source: external.SourceLatitude, LatitudeProjectID: "p"})
+	assert.Error(t, err, "a latitude pool with no IDs is refused")
+}
+
+func captureLog(t *testing.T) *bytes.Buffer {
+	t.Helper()
+	var buf bytes.Buffer
+	prev := slog.Default()
+	slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil)))
+	t.Cleanup(func() { slog.SetDefault(prev) })
+	return &buf
+}
+
+// deploy-vultr.sh waits for this exact text after a restart.
+func TestReportFirstCall_VultrLogTextIsPinned(t *testing.T) {
+	buf := captureLog(t)
+	reportFirstCall(context.Background(), external.ExternalPoolConfig{Name: pool, Source: external.SourceVultr, VultrRegion: "ord"}, nil)
+	assert.Contains(t, buf.String(), `msg="floating using Vultr reserved IPs"`)
+
+	buf.Reset()
+	reportFirstCall(context.Background(), external.ExternalPoolConfig{Name: pool, Source: external.SourceVultr}, errors.New("refused"))
+	assert.Contains(t, buf.String(), `msg="floating: Vultr refused the pool's first call; public addresses will fail until fixed"`)
+}
+
+func TestReportFirstCall_LatitudeLogsItsOwnLine(t *testing.T) {
+	buf := captureLog(t)
+	reportFirstCall(context.Background(), external.ExternalPoolConfig{Name: pool, Source: external.SourceLatitude}, nil)
+	assert.Contains(t, buf.String(), `msg="floating pool ready"`)
+	assert.NotContains(t, buf.String(), "Vultr")
+}
+
+func TestLatitudeProvider_AnUnusableMemberIsAConfigError(t *testing.T) {
+	f := latitudeFake()
+	f.Seed(latitude.ElasticIP{ID: "eip_bgp", Address: netip.MustParseAddr("203.0.113.50"), Mode: "bgp", Family: "IPv4", PrefixLength: 24})
+	p, err := NewLatitudeProvider(f, []string{"eip_1", "eip_bgp"})
+	require.NoError(t, err)
+	_, err = p.Members(context.Background())
+	assert.ErrorContains(t, err, "eip_bgp")
+	assert.ErrorContains(t, err, "routed IPv4 /32")
+}
+
+// A response can put anything in server.id. It becomes the holder the engine logs as
+// from_host after a move, and only a host we configured may be named there.
+func TestHolderPass_UnrecognisedHolderIDNeverReachesTheLog(t *testing.T) {
+	const token = "FAKESECRET"
+	var mu sync.Mutex
+	holder := token // what the API reports as the holder
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		defer mu.Unlock()
+		if r.Method == http.MethodPatch {
+			holder = "sv_bbbbbb"
+			w.WriteHeader(http.StatusAccepted)
+			_, _ = io.WriteString(w, "{}")
+			return
+		}
+		item := `{"id":"eip_aaaaaa","attributes":{"address":"203.0.113.1","status":"active","mode":"routed","family":"IPv4","prefix_length":32,"server":{"id":"` + holder + `"}}}`
+		if strings.HasSuffix(r.URL.Path, "/eip_aaaaaa") {
+			_, _ = io.WriteString(w, `{"data":`+item+`}`)
+			return
+		}
+		_, _ = io.WriteString(w, `{"data":[`+item+`]}`)
+	}))
+	t.Cleanup(srv.Close)
+	keyFile := filepath.Join(t.TempDir(), "token")
+	require.NoError(t, os.WriteFile(keyFile, []byte(token), 0o600))
+	client, err := latitude.New(latitude.Config{BaseURL: srv.URL, KeyFile: keyFile, ProjectID: "p", HTTPClient: srv.Client(), PollInterval: time.Millisecond})
+	require.NoError(t, err)
+	p, err := NewLatitudeProvider(client, []string{"eip_aaaaaa"})
+	require.NoError(t, err)
+
+	n := &node{plugged: map[string]struct{}{}, gateway: map[string]bool{}}
+	alloc, err := New(p, newMemStore(), Config{
+		Pool:       external.ExternalPoolConfig{Name: pool, Source: external.SourceLatitude},
+		HostID:     "sv_bbbbbb",
+		LocalPorts: func(context.Context) (map[string]struct{}, error) { return n.plugged, nil },
+	})
+	require.NoError(t, err)
+	n.alloc = alloc
+	allocENI(t, n, "eni-1")
+	n.plug("eni-1")
+
+	logs := captureLog(t)
+	res, err := alloc.ClaimLocalAddresses(context.Background())
+	require.NoError(t, err)
+	assert.Len(t, res.Claimed, 1)
+	assert.Contains(t, logs.String(), "floating claimed an address", "the move is logged")
+	assert.NotContains(t, logs.String(), token)
+	assert.Contains(t, logs.String(), "unrecognised holder")
+}
+
+// A short server ID that is ours is a real holder: the address is settled, with no move.
+func TestHolderPass_ShortConfiguredHolderIDIsSettled(t *testing.T) {
+	f := latitudeFake()
+	f.Seed(latitude.ElasticIP{ID: "eip_1", Address: netip.MustParseAddr("203.0.113.1"), ServerID: "sv_a"})
+	p, err := NewLatitudeProvider(f, []string{"eip_1"})
+	require.NoError(t, err)
+	n := &node{plugged: map[string]struct{}{}, gateway: map[string]bool{}}
+	alloc, err := New(p, newMemStore(), Config{
+		Pool:       external.ExternalPoolConfig{Name: pool, Source: external.SourceLatitude},
+		HostID:     "sv_a",
+		LocalPorts: func(context.Context) (map[string]struct{}, error) { return n.plugged, nil },
+	})
+	require.NoError(t, err)
+	n.alloc = alloc
+	allocENI(t, n, "eni-1")
+	n.plug("eni-1")
+
+	_, err = alloc.ClaimLocalAddresses(context.Background())
+	require.NoError(t, err)
+	assert.NotContains(t, f.Calls, "move")
+	assert.Equal(t, "sv_a", f.HolderOf("eip_1"))
+}
+
+func TestHolderLabel_NamesOnlyConfiguredHosts(t *testing.T) {
+	a := &PoolAllocator{cfg: Config{HostID: "sv_a", KnownHostIDs: []string{"sv_b"}}}
+	assert.Equal(t, "sv_a", a.holderLabel("sv_a"))
+	assert.Equal(t, "sv_b", a.holderLabel("sv_b"))
+	assert.Empty(t, a.holderLabel(""))
+	assert.Equal(t, "unrecognised holder", a.holderLabel("FAKESECRET"))
+}
@@ -43,6 +43,15 @@ type Provider interface {
 	List(ctx context.Context) ([]Address, error)
 }

+// FixedSet is implemented by a provider whose addresses are bought ahead of
+// time and handed out, not created per allocation. The engine picks a member
+// that has no binding inside the pool's record update, so two nodes cannot
+// take the same one, and never creates, labels or deletes a member.
+type FixedSet interface {
+	// Members returns every address of the set with its current holder.
+	Members(ctx context.Context) ([]Address, error)
+}
+
 var (
 	// ErrNotFound means the address does not exist at the provider.
 	ErrNotFound = errors.New("floating: address not found")
@@ -55,7 +55,7 @@ func (a *PoolAllocator) Reconcile(ctx context.Context) (ReconcileResult, error)
 	var errs []error
 	for _, addr := range live {
 		liveIDs[addr.ID] = struct{}{}
-		if !a.ours(addr) {
+		if a.fixed || !a.ours(addr) {
 			res.Skipped++
 			continue
 		}
@@ -73,7 +73,7 @@ func (a *PoolAllocator) Reconcile(ctx context.Context) (ReconcileResult, error)
 		}
 		slog.WarnContext(ctx, "floating reconcile collected a leaked address",
 			"pool", a.cfg.Pool.Name, "addr_id", addr.ID, "public_ip", addr.IP.String(),
-			"label", addr.Label, "holder", addr.HolderID)
+			"label", addr.Label, "holder", a.holderLabel(addr.HolderID))
 		res.Collected = append(res.Collected, addr.ID)
 	}
 	a.suspects = suspects
@@ -10,36 +10,56 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/network/external"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/host"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/topology"
+	"github.com/mulgadc/spinifex/spinifex/providers/cloud/latitude"
 	"github.com/mulgadc/spinifex/spinifex/providers/cloud/vultr"
 )

 // FromPoolConfig builds a live allocator for one floating pool and proves the
 // credential works before anything depends on it. It does not reconcile; the
 // caller runs that pass. sbAddr is the OVN Southbound address the
-// gateway-chassis question is asked of; empty uses the local socket.
-func FromPoolConfig(ctx context.Context, js jetstream.JetStream, pool external.ExternalPoolConfig, sbAddr string) (*PoolAllocator, error) {
-	if !pool.IsVultr() {
-		return nil, fmt.Errorf("floating: pool %q has source %q, not a floating provider", pool.Name, pool.Source)
-	}
-	client, err := vultr.New(vultr.Config{Region: pool.VultrRegion, KeyFile: pool.VultrKeyFile})
+// gateway-chassis question is asked of; empty uses the local socket. peerHostIDs are
+// the other nodes' provider host IDs, the only holders a log may name.
+func FromPoolConfig(ctx context.Context, js jetstream.JetStream, pool external.ExternalPoolConfig, sbAddr string, peerHostIDs []string) (*PoolAllocator, error) {
+	provider, err := providerFor(pool)
 	if err != nil {
-		return nil, fmt.Errorf("floating: pool %q: %w", pool.Name, err)
+		return nil, err
 	}
-	provider := NewVultrProvider(client)
+	return fromProvider(ctx, js, pool, sbAddr, provider, peerHostIDs)
+}

-	// An allowlisted key refused from this node's address looks ready until the
-	// first launch, so the refusal is surfaced at start with Vultr's reason.
-	if _, err := provider.List(ctx); err != nil {
-		slog.ErrorContext(ctx, "floating: Vultr refused the pool's first call; public addresses will fail until fixed",
-			"pool", pool.Name, "region", pool.VultrRegion, "error", err)
-	} else {
-		slog.InfoContext(ctx, "floating using Vultr reserved IPs", "pool", pool.Name,
-			"region", pool.VultrRegion, "provider_host_id", pool.ProviderHostID, "label_prefix", pool.LabelPrefix)
+// providerFor builds the pool's provider from its source.
+func providerFor(pool external.ExternalPoolConfig) (Provider, error) {
+	switch {
+	case pool.IsVultr():
+		client, err := vultr.New(vultr.Config{Region: pool.VultrRegion, KeyFile: pool.VultrKeyFile})
+		if err != nil {
+			return nil, fmt.Errorf("floating: pool %q: %w", pool.Name, err)
+		}
+		return NewVultrProvider(client), nil
+	case pool.IsLatitude():
+		client, err := latitude.New(latitude.Config{ProjectID: pool.LatitudeProjectID, KeyFile: pool.LatitudeKeyFile})
+		if err != nil {
+			return nil, fmt.Errorf("floating: pool %q: %w", pool.Name, err)
+		}
+		p, err := NewLatitudeProvider(client, pool.LatitudeElasticIPIDs)
+		if err != nil {
+			return nil, fmt.Errorf("floating: pool %q: %w", pool.Name, err)
+		}
+		return p, nil
 	}
+	return nil, fmt.Errorf("floating: pool %q has source %q, not a floating provider", pool.Name, pool.Source)
+}
+
+func fromProvider(ctx context.Context, js jetstream.JetStream, pool external.ExternalPoolConfig, sbAddr string, provider Provider, peerHostIDs []string) (*PoolAllocator, error) {
+	// A refused key looks ready until the first launch, so the refusal is
+	// surfaced at start with the provider's reason.
+	_, err := provider.List(ctx)
+	reportFirstCall(ctx, pool, err)

 	return New(provider, NewKVStore(js), Config{
-		Pool:   pool,
-		HostID: pool.ProviderHostID,
+		Pool:         pool,
+		HostID:       pool.ProviderHostID,
+		KnownHostIDs: peerHostIDs,
 		LocalPorts: func(ctx context.Context) (map[string]struct{}, error) {
 			return host.ListLocalPorts(ctx, host.NewExecRunner())
 		},
@@ -49,3 +69,22 @@ func FromPoolConfig(ctx context.Context, js jetstream.JetStream, pool external.E
 		},
 	})
 }
+
+// reportFirstCall logs the outcome of the pool's first provider call. The Vultr
+// lines are matched by the deploy script, so their text must not change.
+func reportFirstCall(ctx context.Context, pool external.ExternalPoolConfig, err error) {
+	switch {
+	case pool.IsVultr() && err != nil:
+		slog.ErrorContext(ctx, "floating: Vultr refused the pool's first call; public addresses will fail until fixed",
+			"pool", pool.Name, "region", pool.VultrRegion, "error", err)
+	case pool.IsVultr():
+		slog.InfoContext(ctx, "floating using Vultr reserved IPs", "pool", pool.Name,
+			"region", pool.VultrRegion, "provider_host_id", pool.ProviderHostID, "label_prefix", pool.LabelPrefix)
+	case err != nil:
+		slog.ErrorContext(ctx, "floating: the provider refused the pool's first call; public addresses will fail until fixed",
+			"pool", pool.Name, "source", pool.Source, "error", err)
+	default:
+		slog.InfoContext(ctx, "floating pool ready", "pool", pool.Name, "source", pool.Source,
+			"provider_host_id", pool.ProviderHostID)
+	}
+}
@@ -52,6 +52,12 @@ type ExternalPoolConfig struct {
 	VultrRegion  string
 	VultrKeyFile string
 	LabelPrefix  string
+	// Latitude* configure a source="latitude" pool: the project the Elastic IPs
+	// belong to, the file holding the API token, and the fixed set of Elastic IP
+	// IDs the pool hands out. latitude.sh has no labels, so the set is named.
+	LatitudeProjectID    string
+	LatitudeKeyFile      string
+	LatitudeElasticIPIDs []string
 	// ProviderHostID is this node's instance ID at the provider, which a
 	// floating address is attached to. Per node, so it is filled from the
 	// node's own config rather than the shared pool section.
@@ -84,6 +90,17 @@ func (p *ExternalPoolConfig) IsVultr() bool {
 	return p != nil && p.Source == SourceVultr
 }

+// IsLatitude reports whether the pool sources IPs from latitude.sh Elastic IPs.
+func (p *ExternalPoolConfig) IsLatitude() bool {
+	return p != nil && p.Source == SourceLatitude
+}
+
+// IsFloating reports whether the pool's addresses are provider objects moved
+// between nodes by the floating engine.
+func (p *ExternalPoolConfig) IsFloating() bool {
+	return p.IsVultr() || p.IsLatitude()
+}
+
 const (
 	// SourceStatic is the default pool source (inline range math, KV-backed).
 	SourceStatic = "static"
@@ -102,6 +119,9 @@ const (
 	// holder addressed to itself, so like Exoscale there is no private half.
 	// A VM is routed it by MAC; bare metal is ARPed for it on the public NIC.
 	SourceVultr = "vultr"
+	// SourceLatitude hands out a fixed set of latitude.sh Elastic IPs, moved
+	// between servers. A routed Elastic IP arrives addressed to the host's MAC.
+	SourceLatitude = "latitude"

 	// OCIAuthConfigFile authenticates from an API-key config file. The default,
 	// because it is what an operator already has from `oci setup config`.
@@ -31,7 +31,7 @@ type VolumeService interface {
 // Publisher is the provider pair used to reach the new volume's blocks.
 type Publisher interface {
 	PublishVolume(context.Context, ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error)
-	UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) error
+	UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error)
 }

 // Request names the volume to create.
@@ -204,9 +204,10 @@ func (p *Preparer) fill(ctx context.Context, volumeID, image string) error {
 func (p *Preparer) unpublish(ctx context.Context, volumeID string) error {
 	cctx, cancel := p.cleanupContext(ctx)
 	defer cancel()
-	return p.Provider.UnpublishVolume(cctx, ebsprovider.UnpublishVolumeRequest{
+	_, err := p.Provider.UnpublishVolume(cctx, ebsprovider.UnpublishVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: p.NodeID,
 	})
+	return err
 }

 // fail discards the volume when its export is known closed and otherwise
@@ -105,14 +105,17 @@ func (h *harness) PublishVolume(_ context.Context, req ebsprovider.PublishVolume
 	return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: export}, nil
 }

-func (h *harness) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error {
+func (h *harness) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
 	h.record("unpublish " + req.VolumeID + " on " + req.NodeID)
 	h.unpublishCtxOK = ctx.Err() == nil
 	if h.unpublishBlocks {
 		<-ctx.Done()
-		return ctx.Err()
+		return nil, ctx.Err()
 	}
-	return h.unpublishErr
+	if h.unpublishErr != nil {
+		return nil, h.unpublishErr
+	}
+	return &ebsprovider.UnpublishedVolume{}, nil
 }

 func (h *harness) preparer(t *testing.T) *Preparer {
@@ -35,6 +35,12 @@ type Engine struct {
 	// TERMINATING and before the first stage. It records evidence and gates nothing.
 	OnQuiesced func(ctx context.Context, quiescence operatorv1.IAMQuiescence)

+	// Cleanup runs the EC2 platform cleanup. Only Exercise uses it.
+	Cleanup PlatformCleaner
+
+	// OnExercised receives what a non-finalizing run did after quiescence.
+	OnExercised func(ctx context.Context, result operatorv1.ExerciseResult)
+
 	// ec2Inventory is EC2's read-only platform inventory. Only DryRun consults
 	// it, and only to log a comparison; no teardown or Inventory call does.
 	ec2Inventory ec2PlatformInventory
@@ -129,7 +135,7 @@ func (e *Engine) Teardown(ctx context.Context, req Request) (*Result, error) {
 		return nil, fmt.Errorf("quiesce account: %w", err)
 	}
 	if e.OnQuiesced != nil {
-		e.OnQuiesced(ctx, e.evaluateQuiescence(ctx, req, generation))
+		e.OnQuiesced(ctx, e.evaluateQuiescence(ctx, req, generation, false))
 	}
 	e.logger().InfoContext(ctx, "Account teardown started",
 		"accountID", account.AccountID, "accountName", account.AccountName, "force", req.Force)
@@ -338,9 +344,9 @@ func SortReapers(reapers []Reaper) {

 // evaluateQuiescence reads back what the quiesce wrote and, only if the fence
 // is confirmed, asks EC2 for its platform inventory within a short budget.
-func (e *Engine) evaluateQuiescence(ctx context.Context, req Request, generation uint64) operatorv1.IAMQuiescence {
+func (e *Engine) evaluateQuiescence(ctx context.Context, req Request, generation uint64, exercise bool) operatorv1.IAMQuiescence {
 	ev := QuiesceEvidence{
-		OperationID: req.OperationID, Fence: req.Fence, Generation: generation, EvaluatedAt: time.Now(),
+		OperationID: req.OperationID, Fence: req.Fence, Generation: generation, EvaluatedAt: time.Now(), Exercise: exercise,
 	}
 	ev.Account, ev.ReadbackErr = e.Accounts.GetAccount(req.AccountID)
 	if _, confirmed := fenceReadback(ev); confirmed && e.ec2Inventory != nil {
@@ -0,0 +1,207 @@
+package accountlifecycle
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
+	"github.com/nats-io/nats.go"
+)
+
+// PlatformCleaner is the EC2 platform cleanup the exercise runs.
+type PlatformCleaner = accountcleanup.PlatformCleaner
+
+// Exercise quiesces the account, cleans the EC2 platform kinds the evaluation
+// recorded as quiesced, then re-reads the owner inventory and holds it against
+// the cleanup. No legacy stage runs and the account record is never removed.
+func (e *Engine) Exercise(ctx context.Context, req Request) (*operatorv1.ExerciseResult, error) {
+	if _, err := e.checkDeletable(req.AccountID, req.AccountName); err != nil {
+		return nil, err
+	}
+	if req.Force {
+		return nil, errors.New("a non-finalizing run refuses force")
+	}
+	if e.Cleanup == nil || e.ec2Inventory == nil {
+		return nil, errors.New("exercise needs the platform cleanup and inventory")
+	}
+	generation, err := e.Accounts.QuiesceAccount(req.AccountID, req.OperationID, req.Fence)
+	if err != nil {
+		return nil, fmt.Errorf("quiesce account: %w", err)
+	}
+	quiescence := e.evaluateQuiescence(ctx, req, generation, true)
+	if e.OnQuiesced != nil {
+		e.OnQuiesced(ctx, quiescence)
+	}
+
+	result := &operatorv1.ExerciseResult{}
+	if len(quiescence.QuiescedScope) == 0 {
+		result.Skipped = operatorv1.ExerciseSkippedNoQuiescedKind
+	} else {
+		cleaned, err := e.Cleanup.CleanupPlatform(ctx, &accountcleanup.Request{
+			AccountID: req.AccountID, CorrelationRef: req.OperationID, Kinds: quiescence.QuiescedScope,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("platform cleanup: %w", err)
+		}
+		result.Cleanup = platformCleanup(cleaned)
+		result.Accounting = e.accountForRemainder(ctx, req, quiescence.QuiescedScope, result.Cleanup)
+	}
+	if e.OnExercised != nil {
+		e.OnExercised(ctx, *result)
+	}
+	return result, nil
+}
+
+// accountForRemainder re-reads the owner inventory after cleanup and holds it
+// against the cleanup result, per quiesced kind.
+func (e *Engine) accountForRemainder(ctx context.Context, req Request, kinds []string, cleanup *operatorv1.PlatformCleanup) []operatorv1.KindAccounting {
+	budget := ec2PlatformShadowBudget
+	if e.ec2PlatformShadowBudget > 0 {
+		budget = e.ec2PlatformShadowBudget
+	}
+	rereadCtx, cancel := context.WithTimeout(ctx, budget)
+	defer cancel()
+	reply := e.ec2Inventory.PlatformInventory(rereadCtx, req.AccountID, req.OperationID)
+	return AccountForRemainder(kinds, &reply, cleanup)
+}
+
+// AccountForRemainder marks a kind accounted only when its re-read was complete
+// and every record still there is named in the cleanup result with a reason.
+// A record the cleanup reported deleted that is back is unnamed.
+func AccountForRemainder(kinds []string, reread *ec2v1.PlatformInventoryReply, cleanup *operatorv1.PlatformCleanup) []operatorv1.KindAccounting {
+	out := make([]operatorv1.KindAccounting, 0, len(kinds))
+	for _, kind := range kinds {
+		entry := operatorv1.KindAccounting{Kind: kind, Status: operatorv1.AccountingAccounted}
+		status, complete := kindInventory(reread, kind)
+		if !complete {
+			entry.Status, entry.Reason = operatorv1.AccountingUnaccounted, operatorv1.AccountingReasonRereadIncomplete
+			out = append(out, entry)
+			continue
+		}
+		if status == ec2v1.PlatformStatusRecordsFound {
+			unnamed := false
+			for _, item := range reread.Items {
+				if item.Kind != kind {
+					continue
+				}
+				entry.Remaining++
+				unnamed = unnamed || !namedItem(cleanup, item.ResourceID, kind)
+			}
+			for _, count := range reread.Counts {
+				if count.Kind != kind {
+					continue
+				}
+				entry.Remaining += count.Count
+				unnamed = unnamed || !namedCount(cleanup, count)
+			}
+			if unnamed {
+				entry.Status, entry.Reason = operatorv1.AccountingUnaccounted, operatorv1.AccountingReasonUnnamedRemainder
+			}
+		}
+		out = append(out, entry)
+	}
+	return out
+}
+
+func namedItem(cleanup *operatorv1.PlatformCleanup, id, kind string) bool {
+	if cleanup == nil {
+		return false
+	}
+	for _, item := range cleanup.Items {
+		if item.ResourceID == id && item.Kind == kind && item.Disposition != accountcleanup.DispositionDeleted && item.Reason != "" {
+			return true
+		}
+	}
+	return false
+}
+
+func namedCount(cleanup *operatorv1.PlatformCleanup, want ec2v1.PlatformRecordCount) bool {
+	if cleanup == nil {
+		return false
+	}
+	for _, count := range cleanup.Counts {
+		if count.Kind == want.Kind && count.Class == want.Class && count.Disposition != accountcleanup.DispositionDeleted &&
+			count.Reason != "" && count.Count >= want.Count {
+			return true
+		}
+	}
+	return false
+}
+
+func platformCleanup(res *accountcleanup.Result) *operatorv1.PlatformCleanup {
+	out := &operatorv1.PlatformCleanup{Kinds: []operatorv1.CleanupKind{}}
+	for _, kind := range res.Kinds {
+		out.Kinds = append(out.Kinds, operatorv1.CleanupKind{Kind: kind.Kind, Status: kind.Status, IncompleteReasons: kind.IncompleteReasons})
+	}
+	for _, item := range res.Items {
+		out.Items = append(out.Items, operatorv1.CleanupItem{
+			ResourceID: item.ResourceID, Kind: item.Kind, Disposition: item.Disposition, Reason: item.Reason, MemberCount: item.MemberCount,
+		})
+	}
+	for _, count := range res.Counts {
+		out.Counts = append(out.Counts, operatorv1.CleanupCount{
+			Kind: count.Kind, Class: count.Class, Disposition: count.Disposition, Reason: count.Reason, Count: count.Count,
+		})
+	}
+	return out
+}
+
+// InstanceReferenceReader reads the key-pair and placement-group names the
+// account's non-terminated instances carry. It fails closed: a listing that is
+// not complete from every node is an error, never a partial answer.
+type InstanceReferenceReader struct {
+	nc      *nats.Conn
+	nodeIDs []string
+	budget  time.Duration
+}
+
+var _ accountcleanup.InstanceReferenceReader = (*InstanceReferenceReader)(nil)
+
+// NewInstanceReferenceReader builds the reader over the configured node set.
+func NewInstanceReferenceReader(nc *nats.Conn, nodeIDs []string) *InstanceReferenceReader {
+	return &InstanceReferenceReader{nc: nc, nodeIDs: nodeIDs, budget: ec2PlatformShadowBudget}
+}
+
+func (r *InstanceReferenceReader) AccountInstanceReferences(ctx context.Context, accountID string) (*accountcleanup.References, error) {
+	ctx, cancel := context.WithTimeout(ctx, r.budget)
+	defer cancel()
+	out, err := ec2instanceapi.DescribeAccountInstancesComplete(ctx, r.nc, r.nodeIDs, accountID)
+	if err != nil {
+		return nil, err
+	}
+	return instanceReferences(out), nil
+}
+
+func instanceReferences(out *ec2.DescribeInstancesOutput) *accountcleanup.References {
+	refs := &accountcleanup.References{
+		KeyNames: map[string]bool{}, PlacementGroupNames: map[string]bool{}, PlacementGroupIDs: map[string]bool{},
+	}
+	for _, reservation := range out.Reservations {
+		for _, instance := range reservation.Instances {
+			if instance == nil {
+				continue
+			}
+			if instance.State != nil && instance.State.Name != nil && terminalInstanceStates[*instance.State.Name] {
+				continue
+			}
+			if instance.KeyName != nil && *instance.KeyName != "" {
+				refs.KeyNames[*instance.KeyName] = true
+			}
+			if instance.Placement != nil {
+				if instance.Placement.GroupName != nil && *instance.Placement.GroupName != "" {
+					refs.PlacementGroupNames[*instance.Placement.GroupName] = true
+				}
+				if instance.Placement.GroupId != nil && *instance.Placement.GroupId != "" {
+					refs.PlacementGroupIDs[*instance.Placement.GroupId] = true
+				}
+			}
+		}
+	}
+	return refs
+}
@@ -0,0 +1,288 @@
+package accountlifecycle
+
+//test:in-package — the exercise's inventory and cleanup collaborators are
+// unexported seams, and the S7 invariants drive the engine directly.
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type stubCleaner struct {
+	calls  []accountcleanup.Request
+	result *accountcleanup.Result
+	err    error
+	// onCall runs inside the cleanup, so a test can observe what was recorded first.
+	onCall func()
+}
+
+func (s *stubCleaner) CleanupPlatform(_ context.Context, req *accountcleanup.Request) (*accountcleanup.Result, error) {
+	s.calls = append(s.calls, *req)
+	if s.onCall != nil {
+		s.onCall()
+	}
+	return s.result, s.err
+}
+
+// sequencedInventory answers the quiescence evaluation first, then the re-read.
+type sequencedInventory struct {
+	replies []ec2v1.PlatformInventoryReply
+	calls   int
+}
+
+func (s *sequencedInventory) PlatformInventory(context.Context, string, string) ec2v1.PlatformInventoryReply {
+	reply := s.replies[min(s.calls, len(s.replies)-1)]
+	s.calls++
+	return reply
+}
+
+func exerciseEngine(accounts AccountStore, cleaner PlatformCleaner, inventory ec2PlatformInventory) *Engine {
+	engine := quietEngine(accounts)
+	engine.Cleanup = cleaner
+	engine.ec2Inventory = inventory
+	return engine
+}
+
+func exerciseRequest() Request {
+	req := request()
+	req.OperationID, req.Fence = "op-1", 1
+	return req
+}
+
+func cleanResult() *accountcleanup.Result {
+	res := &accountcleanup.Result{AccountID: testAccountID}
+	for _, kind := range ec2v1.PlatformKinds() {
+		res.Kinds = append(res.Kinds, accountcleanup.KindReport{Kind: kind, Status: ec2v1.PlatformStatusObservedClean})
+	}
+	return res
+}
+
+func accountingFor(result *operatorv1.ExerciseResult, kind string) operatorv1.KindAccounting {
+	for _, entry := range result.Accounting {
+		if entry.Kind == kind {
+			return entry
+		}
+	}
+	return operatorv1.KindAccounting{}
+}
+
+// The exercise records the evaluation before it cleans, cleans only the kinds that
+// evaluation recorded as quiesced, and never removes the account record.
+func TestExerciseCleansTheQuiescedKindsAfterRecordingAndKeepsTheAccount(t *testing.T) {
+	accounts := newFakeAccounts()
+	var recorded *operatorv1.IAMQuiescence
+	cleaner := &stubCleaner{result: cleanResult()}
+	cleaner.onCall = func() { assert.NotNil(t, recorded, "quiescence must be recorded before any cleanup") }
+	inventory := &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory()}}
+	engine := exerciseEngine(accounts, cleaner, inventory)
+	engine.OnQuiesced = func(_ context.Context, q operatorv1.IAMQuiescence) { recorded = &q }
+
+	result, err := engine.Exercise(context.Background(), exerciseRequest())
+
+	require.NoError(t, err)
+	require.NotNil(t, recorded)
+	assert.ElementsMatch(t, ec2PlatformKinds, recorded.QuiescedScope)
+	require.Len(t, cleaner.calls, 1)
+	assert.ElementsMatch(t, recorded.QuiescedScope, cleaner.calls[0].Kinds)
+	assert.Equal(t, "op-1", cleaner.calls[0].CorrelationRef)
+	assert.False(t, accounts.deleted, "an exercise never removes the account record")
+	assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
+	assert.Equal(t, 2, inventory.calls, "one evaluation read and one final re-read")
+	for _, kind := range recorded.QuiescedScope {
+		assert.Equal(t, operatorv1.AccountingAccounted, accountingFor(result, kind).Status, kind)
+	}
+}
+
+// The final re-read is held against the cleanup result and the quiescence record
+// is not rewritten from it.
+func TestExerciseAccountsForARemainderOnlyWhenTheCleanupNamedIt(t *testing.T) {
+	cases := map[string]struct {
+		cleanup *accountcleanup.Result
+		reread  ec2v1.PlatformInventoryReply
+		want    string
+		reason  string
+	}{
+		"late write the cleanup never saw": {
+			cleanup: cleanResult(),
+			reread:  rereadWithKeyPair("key-late"),
+			want:    operatorv1.AccountingUnaccounted, reason: operatorv1.AccountingReasonUnnamedRemainder,
+		},
+		"cleanup deleted it but it is back": {
+			cleanup: withItem(cleanResult(), accountcleanup.Item{ResourceID: "key-1", Kind: ec2v1.PlatformKindKeyPair, Disposition: accountcleanup.DispositionDeleted}),
+			reread:  rereadWithKeyPair("key-1"),
+			want:    operatorv1.AccountingUnaccounted, reason: operatorv1.AccountingReasonUnnamedRemainder,
+		},
+		"cleanup blocked it with a reason": {
+			cleanup: withItem(cleanResult(), accountcleanup.Item{ResourceID: "key-1", Kind: ec2v1.PlatformKindKeyPair, Disposition: accountcleanup.DispositionBlocked, Reason: accountcleanup.ReasonInstanceReference}),
+			reread:  rereadWithKeyPair("key-1"),
+			want:    operatorv1.AccountingAccounted,
+		},
+		"re-read incomplete": {
+			cleanup: cleanResult(),
+			reread:  *incompleteKeyPairs(),
+			want:    operatorv1.AccountingUnaccounted, reason: operatorv1.AccountingReasonRereadIncomplete,
+		},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			accounts := newFakeAccounts()
+			inventory := &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory(), tc.reread}}
+			engine := exerciseEngine(accounts, &stubCleaner{result: tc.cleanup}, inventory)
+			var recorded operatorv1.IAMQuiescence
+			engine.OnQuiesced = func(_ context.Context, q operatorv1.IAMQuiescence) { recorded = q }
+
+			result, err := engine.Exercise(context.Background(), exerciseRequest())
+
+			require.NoError(t, err)
+			entry := accountingFor(result, ec2v1.PlatformKindKeyPair)
+			assert.Equal(t, tc.want, entry.Status)
+			assert.Equal(t, tc.reason, entry.Reason)
+			assert.Contains(t, recorded.QuiescedScope, ec2v1.PlatformKindKeyPair, "quiescence is not rewritten from the re-read")
+		})
+	}
+}
+
+func rereadWithKeyPair(id string) ec2v1.PlatformInventoryReply {
+	reply := *completeInventory()
+	reply.Kinds[0].Status = ec2v1.PlatformStatusRecordsFound
+	reply.Items = []ec2v1.PlatformItem{{ResourceID: id, Kind: ec2v1.PlatformKindKeyPair, Disposition: accountcleanup.DispositionBlocked}}
+	return reply
+}
+
+func incompleteKeyPairs() *ec2v1.PlatformInventoryReply {
+	reply := completeInventory()
+	reply.Kinds[0].Status = ec2v1.PlatformStatusScanIncomplete
+	return reply
+}
+
+func withItem(res *accountcleanup.Result, item accountcleanup.Item) *accountcleanup.Result {
+	res.Items = append(res.Items, item)
+	return res
+}
+
+// With no kind quiesced the exercise runs no cleanup and records why.
+func TestExerciseWithNoQuiescedKindRunsNoCleanup(t *testing.T) {
+	accounts := newFakeAccounts()
+	cleaner := &stubCleaner{result: cleanResult()}
+	incomplete := ec2v1.IncompletePlatformInventory(testAccountID, "op-1", ec2v1.PlatformReasonTimeout)
+	engine := exerciseEngine(accounts, cleaner, &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{incomplete}})
+
+	result, err := engine.Exercise(context.Background(), exerciseRequest())
+
+	require.NoError(t, err)
+	assert.Empty(t, cleaner.calls)
+	assert.Equal(t, operatorv1.ExerciseSkippedNoQuiescedKind, result.Skipped)
+	assert.Nil(t, result.Cleanup)
+}
+
+func TestExerciseRefusesForceAndAMissingCollaborator(t *testing.T) {
+	accounts := newFakeAccounts()
+	req := exerciseRequest()
+	req.Force = true
+	_, err := exerciseEngine(accounts, &stubCleaner{}, &stubInventory{}).Exercise(context.Background(), req)
+	require.Error(t, err)
+
+	_, err = exerciseEngine(accounts, nil, &stubInventory{}).Exercise(context.Background(), exerciseRequest())
+	require.Error(t, err)
+	assert.Empty(t, accounts.quiesces, "a refused exercise quiesces nothing")
+}
+
+// ADR-0003:S7: the exercise is an automated actor that deletes customer-visible
+// records, so it acts only on kinds whose quiescence was recorded under a confirmed
+// fence, and a failed cleanup leaves the account where it is.
+func TestS7_ExerciseCleansNothingUnlessItsQuiescenceWasRecordedUnderAConfirmedFence(t *testing.T) {
+	cases := map[string]func(*Engine, *fakeAccounts) *sequencedInventory{
+		"read-back fails": func(e *Engine, a *fakeAccounts) *sequencedInventory {
+			e.Accounts = &readbackFails{fakeAccounts: a}
+			return &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory()}}
+		},
+		"inventory incomplete": func(*Engine, *fakeAccounts) *sequencedInventory {
+			return &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{
+				ec2v1.IncompletePlatformInventory(testAccountID, "op-1", ec2v1.PlatformReasonNoResponder),
+			}}
+		},
+	}
+	for name, setup := range cases {
+		t.Run(name, func(t *testing.T) {
+			accounts := newFakeAccounts()
+			cleaner := &stubCleaner{result: cleanResult()}
+			engine := exerciseEngine(accounts, cleaner, nil)
+			engine.ec2Inventory = setup(engine, accounts)
+
+			_, err := engine.Exercise(context.Background(), exerciseRequest())
+
+			require.NoError(t, err)
+			assert.Empty(t, cleaner.calls, "no kind may be cleaned without a recorded quiescence")
+			assert.False(t, accounts.deleted)
+		})
+	}
+
+	t.Run("only the kinds with a complete inventory are cleaned", func(t *testing.T) {
+		accounts := newFakeAccounts()
+		cleaner := &stubCleaner{result: cleanResult()}
+		partial := completeInventory()
+		partial.Kinds[1].Status = ec2v1.PlatformStatusScanIncomplete
+		engine := exerciseEngine(accounts, cleaner, &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*partial}})
+
+		_, err := engine.Exercise(context.Background(), exerciseRequest())
+
+		require.NoError(t, err)
+		require.Len(t, cleaner.calls, 1)
+		assert.ElementsMatch(t, []string{ec2v1.PlatformKindKeyPair, ec2v1.PlatformKindLaunchTemplate}, cleaner.calls[0].Kinds)
+	})
+
+	t.Run("a failed cleanup returns an error and removes nothing", func(t *testing.T) {
+		accounts := newFakeAccounts()
+		engine := exerciseEngine(accounts, &stubCleaner{err: errors.New("owner down")}, &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory()}})
+
+		_, err := engine.Exercise(context.Background(), exerciseRequest())
+
+		require.Error(t, err)
+		assert.False(t, accounts.deleted)
+	})
+}
+
+// ADR-0003:S7: an ordinary teardown never runs the platform cleanup and never
+// records a quiesced scope, whatever its inventory says.
+func TestS7_OrdinaryTeardownNeverCleansOrRecordsAScope(t *testing.T) {
+	accounts := newFakeAccounts()
+	cleaner := &stubCleaner{result: cleanResult()}
+	engine := exerciseEngine(accounts, cleaner, &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory()}})
+	var recorded operatorv1.IAMQuiescence
+	engine.OnQuiesced = func(_ context.Context, q operatorv1.IAMQuiescence) { recorded = q }
+
+	_, err := engine.Teardown(context.Background(), exerciseRequest())
+
+	require.NoError(t, err)
+	assert.Empty(t, cleaner.calls)
+	assert.Empty(t, recorded.QuiescedScope)
+}
+
+func TestInstanceReferencesSkipTerminalInstancesAndNameKeysAndGroups(t *testing.T) {
+	refs := instanceReferences(&ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{
+		instanceWith("running", "kp-a", "pg-a", "pg-1"),
+		instanceWith("stopped", "kp-b", "", ""),
+		instanceWith("terminated", "kp-gone", "pg-gone", "pg-9"),
+	}}}})
+
+	assert.Equal(t, map[string]bool{"kp-a": true, "kp-b": true}, refs.KeyNames)
+	assert.Equal(t, map[string]bool{"pg-a": true}, refs.PlacementGroupNames)
+	assert.Equal(t, map[string]bool{"pg-1": true}, refs.PlacementGroupIDs)
+}
+
+func instanceWith(state, key, groupName, groupID string) *ec2.Instance {
+	return &ec2.Instance{
+		State:     &ec2.InstanceState{Name: aws.String(state)},
+		KeyName:   aws.String(key),
+		Placement: &ec2.Placement{GroupName: aws.String(groupName), GroupId: aws.String(groupID)},
+	}
+}
@@ -7,17 +7,17 @@ import (
 	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
 )

-// finalInventoryRereadWired says whether anything re-reads the owner inventory
-// before the account record goes. Until it does, a write landing after the
-// reaper stage is orphaned, so no kind may be recorded as quiesced.
-var finalInventoryRereadWired = false
-
 // QuiesceEvidence is what the engine read after QuiesceAccount returned.
 type QuiesceEvidence struct {
 	OperationID string
 	Fence       uint64
 	Generation  uint64

+	// Exercise is set only for a non-finalizing run, which re-reads the owner
+	// inventory after its cleanup and removes no account record. An ordinary
+	// delete re-reads nothing before the record goes, so it records no scope.
+	Exercise bool
+
 	// Account is the IAM record read back after the quiesce.
 	Account     *Account
 	ReadbackErr error
@@ -79,7 +79,8 @@ var otherScopes = []unresolvedEntry{

 // EvaluateQuiescence applies ADR-0008:S9's three conditions to the evidence and
 // records what holds. A fence the read-back does not confirm stops everything
-// else; otherwise every scope is listed, and none is reported quiesced yet.
+// else; otherwise every scope is listed. A kind is quiesced only for an
+// exercise whose owner inventory for it was complete.
 func EvaluateQuiescence(ev QuiesceEvidence) operatorv1.IAMQuiescence {
 	out := operatorv1.IAMQuiescence{
 		OperationID:   ev.OperationID,
@@ -109,7 +110,7 @@ func EvaluateQuiescence(ev QuiesceEvidence) operatorv1.IAMQuiescence {
 		switch {
 		case !complete:
 			out.Unresolved = append(out.Unresolved, writerEntry("ec2-"+kind, operatorv1.ReasonOwnerInventoryIncomplete))
-		case !finalInventoryRereadWired:
+		case !ev.Exercise:
 			out.Unresolved = append(out.Unresolved, writerEntry("ec2-"+kind, operatorv1.ReasonFinalInventoryRereadNotWired))
 		default:
 			out.QuiescedScope = append(out.QuiescedScope, kind)
@@ -72,12 +72,10 @@ func TestEvaluateQuiescenceRecordsEvidenceButQuiescesNothingWithoutAFinalReread(
 	assert.Equal(t, "2026-11-02T00:00:00Z", q.EvaluatedAt)
 }

-// The wiring step flips this switch; the complete kinds are then, and only then, named.
+// Only an exercise, which re-reads the inventory after cleanup, names the complete kinds.
 func TestEvaluateQuiescenceNamesCompleteKindsOnceTheRereadIsWired(t *testing.T) {
-	finalInventoryRereadWired = true
-	t.Cleanup(func() { finalInventoryRereadWired = false })
-
 	ev := confirmedEvidence()
+	ev.Exercise = true
 	ev.Inventory.Kinds[1].Status = ec2v1.PlatformStatusScanIncomplete
 	q := EvaluateQuiescence(ev)

@@ -133,9 +131,8 @@ func TestEvaluateQuiescenceRecordsNothingWhenTheFenceIsNotConfirmed(t *testing.T
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
-			finalInventoryRereadWired = true
-			t.Cleanup(func() { finalInventoryRereadWired = false })
 			ev := confirmedEvidence()
+			ev.Exercise = true
 			tc.mutate(&ev)
 			q := EvaluateQuiescence(ev)

@@ -172,11 +169,11 @@ func TestEveryOtherScopeIsRecordedUnresolvedByName(t *testing.T) {

 // Predastore has no accepted enforcement contract, so the scopes it admits writes
 // for stay unresolved and it is never cited as a consumer, even once the
-// re-read is wired and every L09 kind is complete.
+// run is an exercise and every L09 kind is complete.
 func TestPredastoreIsNeverCitedAndItsScopesNeverQuiesce(t *testing.T) {
-	finalInventoryRereadWired = true
-	t.Cleanup(func() { finalInventoryRereadWired = false })
-	q := EvaluateQuiescence(confirmedEvidence())
+	ev := confirmedEvidence()
+	ev.Exercise = true
+	q := EvaluateQuiescence(ev)

 	for _, scope := range []string{"s3", "ecr"} {
 		got := unresolvedFor(q, scope)
@@ -66,7 +66,10 @@ type accountDeletionJob struct {
 	ClientToken string `json:"clientToken"`
 	State       string `json:"state"`
 	Force       bool   `json:"force"`
-	StartedAt   string `json:"startedAt"`
+	// NonFinalizing marks an exercise: it quiesces and cleans the quiesced EC2
+	// platform kinds, then ends EXERCISED with the account still TERMINATING.
+	NonFinalizing bool   `json:"nonFinalizing,omitempty"`
+	StartedAt     string `json:"startedAt"`
 	// UpdatedAt is the running job's heartbeat. Its absence for long enough is
 	// what lets another gateway take over after a restart.
 	UpdatedAt  string                        `json:"updatedAt"`
@@ -88,6 +91,8 @@ type accountDeletionJob struct {
 	// IAM is the quiescence evaluation made under the current fence. A takeover
 	// starts without one and the new holder records its own.
 	IAM *operatorv1.IAMQuiescence `json:"iam,omitempty"`
+	// Exercise is what a non-finalizing run did after quiescence.
+	Exercise *operatorv1.ExerciseResult `json:"exercise,omitempty"`
 }

 // deletionTakeover records one change of holder.
@@ -99,8 +104,9 @@ type deletionTakeover struct {
 }

 const (
-	takeoverReasonLeaseExpired = "lease_expired"
-	takeoverReasonFailed       = "failed"
+	takeoverReasonLeaseExpired     = "lease_expired"
+	takeoverReasonFailed           = "failed"
+	takeoverReasonExerciseFinished = "exercise_finished"
 )

 // deleteAccount starts a teardown, or answers a dry run inline.
@@ -118,6 +124,11 @@ func (d Deps) deleteAccount(ctx context.Context, body []byte) (any, error) {
 		return nil, errors.New(awserrors.ErrorInternalError)
 	}

+	if req.NonFinalizing && d.PlatformCleaner == nil {
+		slog.Error("DeleteAccount: platform cleanup not available")
+		return nil, errors.New(awserrors.ErrorServiceUnavailable)
+	}
+
 	engine, err := accountteardown.NewClusterEngine(ctx, d.NATS, accountteardown.ClusterNodes{Expected: d.ExpectedNodes, DaemonIDs: d.NodeIDs, GatewayIDs: d.GatewayNodeIDs}, d.Termination, d.BucketStore)
 	if err != nil {
 		slog.Error("DeleteAccount: failed to build teardown engine", "err", err)
@@ -262,6 +273,10 @@ func (d Deps) runAccountDeletion(
 	tracker := &deletionTracker{jobs: jobs, job: job, revision: revision, cancel: cancel}
 	engine.OnStage = tracker.stageFinished
 	engine.OnQuiesced = tracker.quiesced
+	if job.NonFinalizing {
+		engine.Cleanup = d.PlatformCleaner
+		engine.OnExercised = tracker.exercised
+	}

 	go func() {
 		defer cancel()
@@ -269,13 +284,19 @@ func (d Deps) runAccountDeletion(
 		stop := tracker.startHeartbeat(ctx)
 		defer stop()

-		result, err := engine.Teardown(ctx, accountteardown.Request{
+		request := accountteardown.Request{
 			AccountID:   job.AccountID,
 			AccountName: job.AccountName,
 			Force:       job.Force,
 			OperationID: job.OperationID,
 			Fence:       job.Fence,
-		})
+		}
+		if job.NonFinalizing {
+			_, err := engine.Exercise(ctx, request)
+			tracker.finished(ctx, nil, err)
+			return
+		}
+		result, err := engine.Teardown(ctx, request)
 		tracker.finished(ctx, result, err)
 	}()
 }
@@ -308,6 +329,13 @@ func (t *deletionTracker) quiesced(ctx context.Context, quiescence operatorv1.IA
 	t.persist(ctx)
 }

+func (t *deletionTracker) exercised(ctx context.Context, result operatorv1.ExerciseResult) {
+	t.mu.Lock()
+	t.job.Exercise = &result
+	t.mu.Unlock()
+	t.persist(ctx)
+}
+
 func (t *deletionTracker) finished(ctx context.Context, result *accountteardown.Result, err error) {
 	t.mu.Lock()
 	if result != nil {
@@ -316,6 +344,8 @@ func (t *deletionTracker) finished(ctx context.Context, result *accountteardown.
 	if err != nil {
 		t.job.State = operatorv1.DeletionStateFailed
 		t.job.Error = err.Error()
+	} else if t.job.NonFinalizing {
+		t.job.State = operatorv1.DeletionStateExercised
 	} else {
 		t.job.State = operatorv1.DeletionStateCompleted
 	}
@@ -445,8 +475,10 @@ func claimDeletionJob(
 		StartedAt:   now.Format(time.RFC3339),
 		UpdatedAt:   now.Format(time.RFC3339),
 		Fence:       1,
-		Holder:      holder,
-		LeaseUntil:  now.Add(accountDeletionStale).Format(time.RFC3339),
+
+		NonFinalizing: req.NonFinalizing,
+		Holder:        holder,
+		LeaseUntil:    now.Add(accountDeletionStale).Format(time.RFC3339),
 	}
 	record, err := json.Marshal(job)
 	if err != nil {
@@ -478,6 +510,9 @@ func claimDeletionJob(
 	}

 	reason := takeoverReasonFailed
+	if existing.State == operatorv1.DeletionStateExercised {
+		reason = takeoverReasonExerciseFinished
+	}
 	if existing.State == operatorv1.DeletionStateRunning {
 		if !deletionLeaseExpired(existing, now) {
 			return nil, 0, nil, errors.New(awserrors.ErrorOperationInProgress)
@@ -512,8 +547,10 @@ func takeOverDeletionJob(
 		Force:       req.Force,
 		StartedAt:   now.Format(time.RFC3339),
 		UpdatedAt:   now.Format(time.RFC3339),
-		Holder:      holder,
-		LeaseUntil:  now.Add(accountDeletionStale).Format(time.RFC3339),
+
+		NonFinalizing: req.NonFinalizing,
+		Holder:        holder,
+		LeaseUntil:    now.Add(accountDeletionStale).Format(time.RFC3339),
 	}
 	if existing.OperationID != "" {
 		job.OperationID = existing.OperationID
@@ -557,7 +594,7 @@ func replayDeletionJob(
 		return nil, err
 	}
 	if existing.ClientToken == req.ClientToken {
-		if existing.AccountName != req.AccountName {
+		if existing.AccountName != req.AccountName || existing.NonFinalizing != req.NonFinalizing {
 			return nil, errors.New(awserrors.ErrorIdempotentParameterMismatch)
 		}
 		// A RUNNING job whose lease has lapsed has no holder to follow, so the
@@ -703,6 +740,9 @@ func validateDeleteAccountRequest(req *operatorv1.DeleteAccountRequest) error {
 	if !accountIDRE.MatchString(req.AccountID) {
 		return errors.New(awserrors.ErrorInvalidParameterValue)
 	}
+	if req.NonFinalizing && (req.Force || req.DryRun) {
+		return errors.New(awserrors.ErrorInvalidParameterCombination)
+	}
 	if req.DryRun {
 		// A dry run deletes nothing, so it needs neither the name confirmation
 		// nor a token to make it idempotent.
@@ -0,0 +1,133 @@
+package admin
+
+//test:in-package — the exercise runs through the job record, the takeover and
+// the resumer, all of which are unexported.
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const exerciseToken = "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d"
+
+type panicCleaner struct{ calls int }
+
+func (p *panicCleaner) CleanupPlatform(context.Context, *accountcleanup.Request) (*accountcleanup.Result, error) {
+	p.calls++
+	return &accountcleanup.Result{}, nil
+}
+
+func exerciseBody(t *testing.T, accountID, name, token string, mutate func(*operatorv1.DeleteAccountRequest)) []byte {
+	t.Helper()
+	req := operatorv1.DeleteAccountRequest{AccountID: accountID, AccountName: name, ClientToken: token, NonFinalizing: true}
+	if mutate != nil {
+		mutate(&req)
+	}
+	body, err := json.Marshal(req)
+	require.NoError(t, err)
+	return body
+}
+
+func awaitState(t *testing.T, gw Deps, accountID, state string) *accountDeletionJob {
+	t.Helper()
+	require.Eventually(t, func() bool {
+		return describeDeletion(t, gw, accountID).State == state
+	}, 30*time.Second, 100*time.Millisecond)
+	return describeDeletion(t, gw, accountID)
+}
+
+// An exercise ends EXERCISED with the account still TERMINATING, and a later
+// ordinary delete takes the same operation over to finish it.
+func TestNonFinalizingDeleteEndsExercisedAndALaterDeleteTakesItOver(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	gw.PlatformCleaner = &panicCleaner{}
+	account := newTenantAccount(t, gw, "tenant@example.com")
+
+	_, err := gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken, nil))
+	require.NoError(t, err)
+
+	exercised := awaitState(t, gw, account.AccountID, operatorv1.DeletionStateExercised)
+	require.NotNil(t, exercised.IAM)
+	require.NotNil(t, exercised.Exercise)
+	assert.True(t, exercised.NonFinalizing)
+	assert.NotEmpty(t, exercised.FinishedAt)
+	assert.Empty(t, exercised.Stages, "no legacy stage runs")
+	stored, err := gw.IAM.GetAccount(account.AccountID)
+	require.NoError(t, err)
+	assert.Equal(t, "TERMINATING", stored.Status)
+
+	_, err = gw.deleteAccount(t.Context(), deleteAccountBody(t, account.AccountID, "tenant@example.com", deleteClientToken))
+	require.NoError(t, err)
+
+	done := awaitState(t, gw, account.AccountID, operatorv1.DeletionStateCompleted)
+	assert.Equal(t, exercised.OperationID, done.OperationID)
+	assert.Equal(t, uint64(2), done.Fence)
+	require.Len(t, done.Takeovers, 1)
+	assert.Equal(t, takeoverReasonExerciseFinished, done.Takeovers[0].Reason)
+	assert.Nil(t, done.Exercise, "the takeover starts a fresh run")
+	assert.Equal(t, uint64(2), done.IAM.Fence)
+	assert.Empty(t, done.IAM.QuiescedScope, "an ordinary delete records no scope")
+}
+
+func TestNonFinalizingIsPartOfTheIdempotencyKeyAndRefusesForce(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	gw.PlatformCleaner = &panicCleaner{}
+	account := newTenantAccount(t, gw, "tenant@example.com")
+
+	_, err := gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken,
+		func(r *operatorv1.DeleteAccountRequest) { r.Force = true }))
+	require.EqualError(t, err, awserrors.ErrorInvalidParameterCombination)
+
+	_, err = gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken, nil))
+	require.NoError(t, err)
+	awaitState(t, gw, account.AccountID, operatorv1.DeletionStateExercised)
+
+	_, err = gw.deleteAccount(t.Context(), deleteAccountBody(t, account.AccountID, "tenant@example.com", exerciseToken))
+	require.EqualError(t, err, awserrors.ErrorIdempotentParameterMismatch, "the same token cannot turn an exercise into a delete")
+
+	again, err := gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken, nil))
+	require.NoError(t, err)
+	assert.Equal(t, operatorv1.DeletionStateExercised, again.(*DeleteAccountResponse).State, "a repeated exercise is answered from the record")
+}
+
+func TestNonFinalizingWithoutACleanupIsRefusedBeforeAnyJobExists(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	account := newTenantAccount(t, gw, "tenant@example.com")
+
+	_, err := gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken, nil))
+
+	require.EqualError(t, err, awserrors.ErrorServiceUnavailable)
+	jobs, err := gw.accountDeletionStore(t.Context())
+	require.NoError(t, err)
+	stored, err := readDeletionJob(t.Context(), jobs, account.AccountID)
+	require.NoError(t, err)
+	assert.Nil(t, stored)
+}
+
+// A resumed exercise stays an exercise: the resumer must not turn a recorded
+// non-finalizing decision into a deletion.
+func TestResumerKeepsAnAbandonedExerciseNonFinalizing(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	gw.PlatformCleaner = &panicCleaner{}
+	account := newTenantAccount(t, gw, "tenant@example.com")
+	job := abandonedJob(account.AccountID, "tenant@example.com")
+	job.NonFinalizing = true
+	storeDeletionJob(t, gw, job)
+
+	require.Equal(t, 1, gw.resumeDeletions(t.Context(), time.Now().UTC()))
+
+	done := awaitState(t, gw, account.AccountID, operatorv1.DeletionStateExercised)
+	assert.Equal(t, uint64(2), done.Fence)
+	assert.True(t, done.NonFinalizing)
+	stored, err := gw.IAM.GetAccount(account.AccountID)
+	require.NoError(t, err)
+	assert.Equal(t, "TERMINATING", stored.Status, "the account record is never removed")
+}
@@ -110,6 +110,8 @@ func (d Deps) resumeDeletion(
 		AccountName: existing.AccountName,
 		ClientToken: existing.ClientToken,
 		Force:       existing.Force,
+
+		NonFinalizing: existing.NonFinalizing,
 	}
 	job, newRevision, _, err := takeOverDeletionJob(ctx, jobs, req, d.holderID(), existing, revision, takeoverReasonLeaseExpired, now)
 	if err != nil {
@@ -45,6 +45,7 @@ func TestS7_DeletionResumerActsOnlyOnAnAcceptedOperationWithoutALiveLease(t *tes
 		},
 		"failed job":    func(j *accountDeletionJob) { j.State = operatorv1.DeletionStateFailed },
 		"completed job": func(j *accountDeletionJob) { j.State = operatorv1.DeletionStateCompleted },
+		"exercised job": func(j *accountDeletionJob) { j.State = operatorv1.DeletionStateExercised },
 		"legacy record": func(j *accountDeletionJob) { j.OperationID, j.Fence, j.LeaseUntil = "", 0, "" },
 	}
 	for name, mutate := range cases {
@@ -76,7 +76,7 @@ func ImportImage(ctx context.Context, provider ebsprovider.EBSProvider, opts Imp

 	// Unpublish regardless: a volume left published cannot be snapshotted or
 	// attached, and on the failure path it would strand the export too.
-	if err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
+	if _, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(),
 		VolumeID:  opts.VolumeID,
 		NodeID:    opts.NodeID,
@@ -49,7 +49,7 @@ func (p *recordingProvider) PublishVolume(ctx context.Context, req ebsprovider.P
 	return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: export}, nil
 }

-func (p *recordingProvider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error {
+func (p *recordingProvider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
 	p.calls = append(p.calls, "unpublish")
 	return p.MemoryProvider.UnpublishVolume(ctx, req)
 }
@@ -58,7 +58,10 @@ type Deps struct {
 	IAM iamdomain.IAMService
 	// Termination is the concrete IAM service, the only holder of the capability
 	// to quiesce and remove an account. DeleteAccount needs it.
-	Termination       accountteardown.IAMBackend
+	Termination accountteardown.IAMBackend
+	// PlatformCleaner is the in-process EC2 platform cleanup a non-finalizing
+	// DeleteAccount runs. Nil refuses that request.
+	PlatformCleaner   accountteardown.PlatformCleaner
 	Quota             *quota.Service
 	ExpectedNodes     int
 	NodeIDs           []string
@@ -61,6 +61,8 @@ func init() {
 	accountCmd.AddCommand(accountDeleteCmd)

 	accountDeleteCmd.Flags().Bool("dry-run", false, "Report what would be deleted and change nothing")
+	accountDeleteCmd.Flags().Bool("non-finalizing", false, "Quiesce and clean the EC2 platform kinds, then stop with the account still TERMINATING")
+	_ = accountDeleteCmd.Flags().MarkHidden("non-finalizing")
 	accountDeleteCmd.Flags().Bool("force", false, "Escalate past state guards for resources that will not delete")
 	accountDeleteCmd.Flags().Bool("yes", false, "Skip the interactive confirmation (requires --name)")
 	accountDeleteCmd.Flags().String("name", "", "Account name, which must match the stored record")
@@ -24,6 +24,7 @@ var accountDeletePollInterval = 10 * time.Second
 func runAccountDeleteRemote(cmd *cobra.Command, accountID string) error {
 	dryRun, _ := cmd.Flags().GetBool("dry-run")
 	force, _ := cmd.Flags().GetBool("force")
+	nonFinalizing, _ := cmd.Flags().GetBool("non-finalizing")
 	assumeYes, _ := cmd.Flags().GetBool("yes")
 	name, _ := cmd.Flags().GetString("name")
 	clientToken, _ := cmd.Flags().GetString("client-token")
@@ -66,6 +67,7 @@ func runAccountDeleteRemote(cmd *cobra.Command, accountID string) error {

 	started, err := deleteAccountRemote(ctx, target, operatorv1.DeleteAccountRequest{
 		AccountID: accountID, AccountName: name, ClientToken: clientToken, Force: force,
+		NonFinalizing: nonFinalizing,
 	})
 	if err != nil {
 		var adminErr *adminError
@@ -101,6 +103,9 @@ func followAccountDeletion(ctx context.Context, target adminTarget, accountID st
 		case operatorv1.DeletionStateCompleted:
 			fmt.Printf("\nAccount %s deleted.\n", accountID)
 			return nil
+		case operatorv1.DeletionStateExercised:
+			fmt.Printf("\nAccount %s exercised and left TERMINATING; a normal delete finishes it.\n", accountID)
+			return nil
 		case operatorv1.DeletionStateFailed:
 			fmt.Fprintf(os.Stderr, "\nThe account is still TERMINATING so its leftovers keep an owner.\n")
 			return fmt.Errorf("teardown failed: %s", job.Error)
@@ -1748,3 +1748,31 @@ func TestApplyNetworkConfig_PropagatesPoolDNSServersWithoutSelf(t *testing.T) {
 	require.Len(t, settings.Pools, 1)
 	assert.Equal(t, []string{"1.1.1.1", "10.0.0.6", "8.8.8.8"}, settings.Pools[0].DNSServers)
 }
+
+// A generated config writes no [nodes.<peer>.ebs] stanza, so a node switched to
+// a non-default provider must still load: an absent peer provider is unknown.
+func TestSpinifexTomlTemplate_NonDefaultProviderLoadsBesidePeers(t *testing.T) {
+	allNodes := map[string]formation.NodeInfo{
+		"node1": {Name: "node1", BindIP: "10.0.0.1", AdvertiseIP: "192.168.1.31"},
+		"node2": {Name: "node2", BindIP: "10.0.0.2", AdvertiseIP: "192.168.1.32"},
+		"node3": {Name: "node3", BindIP: "10.0.0.3", AdvertiseIP: "192.168.1.33"},
+	}
+	settings := multinodeNorthstarSettings("/etc/spinifex/northstar/northstar.toml")
+	settings.RemoteNodes = buildRemoteNodes(allNodes, "node1", settings.NorthstarConfigPath)
+	require.Len(t, settings.RemoteNodes, 2)
+
+	path := filepath.Join(t.TempDir(), "spinifex.toml")
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+	rendered, err := os.ReadFile(path)
+	require.NoError(t, err)
+	const defaultLine = "provider = \"viperblockd\"\n"
+	require.Equal(t, 1, strings.Count(string(rendered), defaultLine), "the template must set the provider for its own node only")
+	switched := strings.Replace(string(rendered), defaultLine, "provider = \"qemunbd\"\n", 1)
+	require.NoError(t, os.WriteFile(path, []byte(switched), 0o600))
+
+	cfg, err := config.LoadConfig(path)
+	require.NoError(t, err)
+	assert.Equal(t, config.EBSProviderQEMUNBD, cfg.Nodes["node1"].EBS.ResolvedProvider())
+	assert.Contains(t, cfg.Nodes, "node2")
+	assert.Contains(t, cfg.Nodes, "node3")
+}
@@ -136,6 +136,9 @@ encryption_key_file = "{{.EncryptionKeyFile}}"
 {{- end}}

 [nodes.{{.Node}}.ebs]
+# provider: every node in a cluster must name the same one, and the daemon
+# refuses to start on a provider other than the one the cluster recorded.
+# Changing it needs the cluster wiped and reformed.
 provider = "viperblockd"
 # Per-volume export tunables. Both are applied per nbdkit process and there is
 # one of those per volume, so the host cost of each is the value times the
@@ -21,6 +21,7 @@ Single source of truth for the current schema version of every config file Spini
 |---|---|---|
 | `spinifex-instance-state` | `5` | `daemon.InstanceStateBucketVersion` |
 | `spinifex-cluster-state` | `1` | `daemon.ClusterStateBucketVersion` |
+| `spinifex-cluster-config` | `1` | `daemon.ClusterConfigBucketVersion` |
 | `spinifex-terminated-instances` | `3` | `daemon.TerminatedInstanceBucketVersion` |

 ## Registered migrations
@@ -0,0 +1,464 @@
+// Package latitude is the narrow slice of the latitude.sh API that floating
+// public addresses need: reading a project's Elastic IPs and moving one to
+// another server.
+//
+// It is a direct net/http client, like the Vultr one. Request and response
+// shapes follow the published OpenAPI spec and the Terraform provider; no call
+// has been made against the live API, so the tests use bodies written from the
+// spec, not captured responses.
+package latitude
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand/v2"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"os"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// DefaultBaseURL is the public API.
+const DefaultBaseURL = "https://api.latitude.sh"
+
+// DefaultKeyFile holds the node's API token, readable by the daemon's user only.
+const DefaultKeyFile = "/etc/spinifex/latitude/api-token"
+
+// DefaultCallTimeout bounds one HTTP round trip.
+const DefaultCallTimeout = 15 * time.Second
+
+const (
+	defaultAttempts = 3
+	// listPageSize is requested once: the listing ignores page[number], and a
+	// team holds at most a handful of Elastic IPs.
+	listPageSize = 100
+	// DefaultMoveTimeout bounds how long a move waits for the address to be active again.
+	DefaultMoveTimeout = 2 * time.Minute
+	// DefaultPollInterval is the gap between status reads while a move settles.
+	DefaultPollInterval = 3 * time.Second
+)
+
+var (
+	// ErrNotFound means the Elastic IP or server does not exist.
+	ErrNotFound = errors.New("latitude: resource not found")
+	// ErrMoveRefused means the API will not move the address to that server:
+	// another project, another site, or an incompatible network. Retrying is pointless.
+	ErrMoveRefused = errors.New("latitude: move refused")
+	// ErrNotActive means the address is still settling from an earlier change, so no move
+	// was attempted; the caller tries again on its next pass.
+	ErrNotActive = errors.New("latitude: Elastic IP not active")
+	// ErrNotSettled means the address did not return to active within the move timeout.
+	ErrNotSettled = errors.New("latitude: Elastic IP did not become active")
+	// ErrUnauthorized means the token was refused.
+	ErrUnauthorized = errors.New("latitude: API token refused")
+	// ErrUnavailable means the API was throttled or failing; retry later.
+	ErrUnavailable = errors.New("latitude: API unavailable")
+)
+
+// APIError is a non-2xx response, carrying latitude.sh's own code and message.
+type APIError struct {
+	Status  int
+	Code    string
+	Message string
+	kind    error
+}
+
+// Error reports the status and, when it is a well-formed code, latitude.sh's error code.
+// The response body is never included: an upstream error can echo a credential.
+func (e *APIError) Error() string {
+	if e.Code != "" {
+		return fmt.Sprintf("latitude: HTTP %d %s", e.Status, e.Code)
+	}
+	return fmt.Sprintf("latitude: HTTP %d %s", e.Status, e.Message)
+}
+
+// Unwrap exposes the classified sentinel, if any, to errors.Is.
+func (e *APIError) Unwrap() error { return e.kind }
+
+// Status values of an Elastic IP.
+const (
+	StatusActive = "active"
+	StatusError  = "error"
+)
+
+// ElasticIP is one Elastic IP.
+type ElasticIP struct {
+	ID           string
+	Address      netip.Addr
+	Status       string
+	Mode         string
+	Family       string
+	PrefixLength int
+	// ServerID is empty when the address is unassigned or its server is not
+	// active, which the API does not distinguish.
+	ServerID string
+}
+
+// Unusable returns why the address cannot be handed out by a floating pool, or
+// empty. Only a routed IPv4 /32 can be moved between servers.
+func (e ElasticIP) Unusable() string {
+	if e.Mode != "routed" || e.Family != "IPv4" || e.PrefixLength != 32 {
+		return fmt.Sprintf("mode %s, family %s, prefix length %d: only a routed IPv4 /32 can be moved",
+			safe(e.Mode, "routed", "bgp"), safe(e.Family, "IPv4", "IPv6"), e.PrefixLength)
+	}
+	return ""
+}
+
+// Config selects the endpoint, credential and project.
+type Config struct {
+	BaseURL     string
+	KeyFile     string
+	ProjectID   string
+	CallTimeout time.Duration
+	HTTPClient  *http.Client
+	// Attempts is the total tries for a retryable failure; zero means the default.
+	Attempts int
+	// Backoff is the first retry delay; it doubles with jitter. Zero means 500ms.
+	Backoff time.Duration
+	// MoveTimeout and PollInterval pace the wait after a move; zero means the defaults.
+	MoveTimeout  time.Duration
+	PollInterval time.Duration
+}
+
+// Client implements the Elastic IP calls against one project.
+type Client struct {
+	cfg  Config
+	http *http.Client
+}
+
+// New builds a client. It makes no call; ListElasticIPs proves the token works.
+func New(cfg Config) (*Client, error) {
+	if cfg.ProjectID == "" {
+		return nil, errors.New("latitude: project id is required")
+	}
+	if cfg.BaseURL == "" {
+		cfg.BaseURL = DefaultBaseURL
+	}
+	cfg.BaseURL = strings.TrimRight(cfg.BaseURL, "/")
+	if cfg.KeyFile == "" {
+		cfg.KeyFile = DefaultKeyFile
+	}
+	if cfg.CallTimeout <= 0 {
+		cfg.CallTimeout = DefaultCallTimeout
+	}
+	if cfg.Attempts <= 0 {
+		cfg.Attempts = defaultAttempts
+	}
+	if cfg.Backoff <= 0 {
+		cfg.Backoff = 500 * time.Millisecond
+	}
+	if cfg.MoveTimeout <= 0 {
+		cfg.MoveTimeout = DefaultMoveTimeout
+	}
+	if cfg.PollInterval <= 0 {
+		cfg.PollInterval = DefaultPollInterval
+	}
+	hc := &http.Client{}
+	if cfg.HTTPClient != nil {
+		clone := *cfg.HTTPClient
+		hc = &clone
+	}
+	// The API never redirects a call we make. Following a 307 or 308 would replay a
+	// PATCH body, a second write the caller did not authorise.
+	hc.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
+	return &Client{cfg: cfg, http: hc}, nil
+}
+
+type elasticIPJSON struct {
+	ID         *string `json:"id"`
+	Attributes struct {
+		Address      string `json:"address"`
+		Status       string `json:"status"`
+		Mode         string `json:"mode"`
+		Family       string `json:"family"`
+		PrefixLength int    `json:"prefix_length"`
+		Server       *struct {
+			ID string `json:"id"`
+		} `json:"server"`
+	} `json:"attributes"`
+}
+
+// toElasticIP keeps every ID exactly as the API sent it. IDs are opaque: they are compared, never
+// inspected, and the caller decides which may be logged.
+func (r elasticIPJSON) toElasticIP() (ElasticIP, error) {
+	if r.ID == nil || *r.ID == "" {
+		return ElasticIP{}, errors.New("latitude: Elastic IP with no id in response")
+	}
+	addr, err := netip.ParseAddr(r.Attributes.Address)
+	if err != nil {
+		return ElasticIP{}, errors.New("latitude: Elastic IP response has an unparseable address")
+	}
+	out := ElasticIP{
+		ID: *r.ID, Address: addr, Status: r.Attributes.Status,
+		Mode: r.Attributes.Mode, Family: r.Attributes.Family, PrefixLength: r.Attributes.PrefixLength,
+	}
+	if r.Attributes.Server != nil {
+		if r.Attributes.Server.ID != "" {
+			out.ServerID = r.Attributes.Server.ID
+		}
+	}
+	return out, nil
+}
+
+// GetElasticIP reads one address, including which server holds it.
+func (c *Client) GetElasticIP(ctx context.Context, id string) (ElasticIP, error) {
+	var out struct {
+		Data elasticIPJSON `json:"data"`
+	}
+	if err := c.do(ctx, http.MethodGet, "/elastic_ips/"+url.PathEscape(id), nil, &out); err != nil {
+		return ElasticIP{}, err
+	}
+	return out.Data.toElasticIP()
+}
+
+// ListElasticIPs returns every address of the project. The listing is read in
+// one page; a full page is an error, since page[number] is ignored and the rest
+// would be silently missing.
+func (c *Client) ListElasticIPs(ctx context.Context) ([]ElasticIP, error) {
+	q := url.Values{"filter[project]": {c.cfg.ProjectID}, "page[size]": {strconv.Itoa(listPageSize)}}
+	var out struct {
+		Data []elasticIPJSON `json:"data"`
+	}
+	if err := c.do(ctx, http.MethodGet, "/elastic_ips?"+q.Encode(), nil, &out); err != nil {
+		return nil, err
+	}
+	if len(out.Data) >= listPageSize {
+		return nil, fmt.Errorf("latitude: Elastic IP listing returned a full page of %d; refusing a possibly truncated set", listPageSize)
+	}
+	all := make([]ElasticIP, 0, len(out.Data))
+	for _, r := range out.Data {
+		ip, err := r.toElasticIP()
+		if err != nil {
+			return nil, err
+		}
+		all = append(all, ip)
+	}
+	return all, nil
+}
+
+// MoveElasticIP points id at serverID and waits until it is active there.
+// It makes at most one write per call: the caller checked that it still owns the
+// address just before calling, and a second write after a wait would outlive that
+// check. An address that is not active is reported as ErrNotActive without a
+// write, and the caller tries again on its next pass. After the write only reads
+// follow, until the move settles or the move timeout passes.
+func (c *Client) MoveElasticIP(ctx context.Context, id, serverID string) error {
+	cur, err := c.GetElasticIP(ctx, id)
+	if err != nil {
+		return err
+	}
+	if why := cur.Unusable(); why != "" {
+		return fmt.Errorf("%w: %s: %s", ErrMoveRefused, id, why)
+	}
+	switch {
+	case cur.Status == StatusError:
+		return fmt.Errorf("%w: %s is in the error state", ErrMoveRefused, id)
+	case cur.Status == StatusActive && cur.ServerID == serverID:
+		return nil
+	case cur.Status != StatusActive:
+		return fmt.Errorf("%w: %s is %s", ErrNotActive, id, safe(cur.Status, "pending", "configuring", "moving", "releasing"))
+	}
+	body := map[string]any{"data": map[string]any{
+		"type":       "elastic_ips",
+		"attributes": map[string]string{"server_id": serverID},
+	}}
+	// Exactly one attempt: a retry after a throttle, a 5xx or a lost connection would be a second
+	// write after the caller's ownership check, and the next engine pass revalidates anyway.
+	payload, err := json.Marshal(body)
+	if err != nil {
+		return fmt.Errorf("latitude: encode move: %w", err)
+	}
+	if err := c.once(ctx, http.MethodPatch, "/elastic_ips/"+url.PathEscape(id), payload, nil); err != nil {
+		return err
+	}
+	return c.awaitActive(ctx, id, serverID)
+}
+
+// awaitActive reads the address until it is active on serverID. It writes nothing.
+func (c *Client) awaitActive(ctx context.Context, id, serverID string) error {
+	ctx, cancel := context.WithTimeout(ctx, c.cfg.MoveTimeout)
+	defer cancel()
+	for {
+		cur, err := c.GetElasticIP(ctx, id)
+		if err != nil {
+			if ctx.Err() != nil {
+				return fmt.Errorf("%w: %s: %w", ErrNotSettled, id, err)
+			}
+			return err
+		}
+		if cur.Status == StatusError {
+			return fmt.Errorf("%w: %s went to the error state", ErrMoveRefused, id)
+		}
+		if cur.Status == StatusActive && cur.ServerID == serverID {
+			return nil
+		}
+		if err := c.pause(ctx); err != nil {
+			return fmt.Errorf("%w: %s: %w", ErrNotSettled, id, err)
+		}
+	}
+}
+
+func (c *Client) pause(ctx context.Context) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-time.After(c.cfg.PollInterval):
+		return nil
+	}
+}
+
+// do runs one call, retrying throttles and server failures. Writes are retried
+// too: a repeated move to the same server is harmless.
+func (c *Client) do(ctx context.Context, method, path string, in, out any) error {
+	var payload []byte
+	if in != nil {
+		var err error
+		if payload, err = json.Marshal(in); err != nil {
+			return fmt.Errorf("latitude: encode %s %s: %w", method, path, err)
+		}
+	}
+	delay := c.cfg.Backoff
+	for attempt := 1; ; attempt++ {
+		err := c.once(ctx, method, path, payload, out)
+		if err == nil || !errors.Is(err, ErrUnavailable) || attempt >= c.cfg.Attempts {
+			return err
+		}
+		wait := delay/2 + rand.N(delay) //#nosec G404 - backoff jitter, not a secret
+		select {
+		case <-ctx.Done():
+			return errors.Join(err, ctx.Err())
+		case <-time.After(wait):
+		}
+		delay *= 2
+	}
+}
+
+// once makes a single round trip. The token is read on every call, so rotating
+// it in place needs no restart.
+func (c *Client) once(ctx context.Context, method, path string, payload []byte, out any) error {
+	key, err := c.key()
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(ctx, c.cfg.CallTimeout)
+	defer cancel()
+
+	var body io.Reader
+	if payload != nil {
+		body = bytes.NewReader(payload)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, c.cfg.BaseURL+path, body)
+	if err != nil {
+		return fmt.Errorf("latitude: build %s %s: %w", method, path, err)
+	}
+	req.Header.Set("Authorization", "Bearer "+key)
+	req.Header.Set("Accept", "application/json")
+	if payload != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return fmt.Errorf("%w: %s %s: %s", ErrUnavailable, method, path, redact(err.Error(), key))
+	}
+	defer func() { _ = resp.Body.Close() }()
+	raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if err != nil {
+		return fmt.Errorf("%w: read %s %s: %s", ErrUnavailable, method, path, redact(err.Error(), key))
+	}
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return classify(resp.StatusCode, raw)
+	}
+	if out == nil || len(raw) == 0 {
+		return nil
+	}
+	if err := json.Unmarshal(raw, out); err != nil {
+		return fmt.Errorf("latitude: decode %s %s: %s", method, path, redact(err.Error(), key))
+	}
+	return nil
+}
+
+func (c *Client) key() (string, error) {
+	raw, err := os.ReadFile(c.cfg.KeyFile)
+	if err != nil {
+		return "", fmt.Errorf("latitude: read API token: %w", err)
+	}
+	key := strings.TrimSpace(string(raw))
+	if key == "" {
+		return "", fmt.Errorf("latitude: API token file %s is empty", c.cfg.KeyFile)
+	}
+	return key, nil
+}
+
+// knownCodes are the error codes latitude.sh documents for Elastic IPs. Any other
+// code is dropped, so a response cannot put its own text into an error.
+var knownCodes = map[string]struct{}{
+	"ELASTIC_IP_NOT_ACTIVE":        {},
+	"ELASTIC_IP_NOT_RELEASABLE":    {},
+	"ELASTIC_IP_LIMIT_REACHED":     {},
+	"ELASTIC_IP_HAS_BGP_SESSIONS":  {},
+	"IP_ALLOCATION_FAILED":         {},
+	"SERVER_NOT_IN_PROJECT":        {},
+	"SITE_MISMATCH":                {},
+	"SITE_NOT_SUPPORTED":           {},
+	"SERVER_NETWORK_INCOMPATIBLE":  {},
+	"BGP_ELASTIC_IP_NOT_UPDATABLE": {},
+}
+
+// safe returns v when it is one of the documented values, else "unknown". Values
+// read from a response never reach an error otherwise.
+func safe(v string, known ...string) string {
+	if slices.Contains(known, v) {
+		return v
+	}
+	return "unknown"
+}
+
+// classify maps a non-2xx response onto the sentinels callers branch on. Only the
+// status and a well-formed error code are kept; the body itself never reaches an error.
+func classify(status int, raw []byte) error {
+	var body struct {
+		Errors []struct {
+			Code string `json:"code"`
+		} `json:"errors"`
+	}
+	_ = json.Unmarshal(raw, &body)
+	e := &APIError{Status: status, Message: http.StatusText(status)}
+	if len(body.Errors) > 0 {
+		if _, ok := knownCodes[body.Errors[0].Code]; ok {
+			e.Code = body.Errors[0].Code
+		}
+	}
+	switch {
+	case status == http.StatusUnauthorized || status == http.StatusForbidden:
+		e.kind = ErrUnauthorized
+	case status == http.StatusNotFound:
+		e.kind = ErrNotFound
+	case status == http.StatusTooManyRequests || status >= 500:
+		e.kind = ErrUnavailable
+	case e.Code == "ELASTIC_IP_NOT_ACTIVE":
+		e.kind = ErrNotActive
+	case status == http.StatusUnprocessableEntity:
+		switch e.Code {
+		case "SERVER_NOT_IN_PROJECT", "SITE_MISMATCH", "SITE_NOT_SUPPORTED", "SERVER_NETWORK_INCOMPATIBLE", "BGP_ELASTIC_IP_NOT_UPDATABLE":
+			e.kind = ErrMoveRefused
+		}
+	}
+	return e
+}
+
+// redact removes the token from a message that might carry it.
+func redact(msg, token string) string {
+	if token == "" {
+		return msg
+	}
+	return strings.ReplaceAll(msg, token, "[redacted]")
+}
@@ -0,0 +1,441 @@
+package latitude
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// api serves canned responses in order per "METHOD path", and records bodies and tokens.
+type api struct {
+	t       *testing.T
+	mu      sync.Mutex
+	replies map[string][]reply
+	bodies  []string
+	auth    []string
+	queries []string
+}
+
+const testToken = "FAKESECRET"
+
+type reply struct {
+	status int
+	body   string
+}
+
+func eip(id, addr, status, server string) string {
+	srv := "null"
+	if server != "" {
+		srv = fmt.Sprintf(`{"id":%q}`, server)
+	}
+	idv := "null"
+	if id != "" {
+		idv = fmt.Sprintf("%q", id)
+	}
+	return fmt.Sprintf(`{"id":%s,"type":"elastic_ips","attributes":{"address":%q,"status":%q,"mode":"routed","family":"IPv4","prefix_length":32,"server":%s}}`, idv, addr, status, srv)
+}
+
+// bgp is the documented shape of a BGP-mode address, which cannot be moved.
+func bgp(id, addr string) string {
+	return fmt.Sprintf(`{"id":%q,"type":"elastic_ips","attributes":{"address":%q,"status":"active","mode":"bgp","family":"IPv4","prefix_length":24,"server":null}}`, id, addr)
+}
+
+func one(id, addr, status, server string) string {
+	return `{"data":` + eip(id, addr, status, server) + `}`
+}
+
+func errBody(code string) string {
+	return fmt.Sprintf(`{"errors":[{"code":%q,"title":"t","detail":"d %s"}]}`, code, code)
+}
+
+func newClient(t *testing.T, replies map[string][]reply) (*api, *Client) {
+	t.Helper()
+	a := &api{t: t, replies: replies}
+	srv := httptest.NewServer(a)
+	t.Cleanup(srv.Close)
+	keyFile := filepath.Join(t.TempDir(), "token")
+	require.NoError(t, os.WriteFile(keyFile, []byte(testToken+"\n"), 0o600))
+	c, err := New(Config{
+		BaseURL: srv.URL, KeyFile: keyFile, ProjectID: "proj_1",
+		Backoff: time.Millisecond, PollInterval: time.Millisecond, MoveTimeout: time.Second,
+	})
+	require.NoError(t, err)
+	return a, c
+}
+
+func (a *api) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	key := r.Method + " " + r.URL.Path
+	a.auth = append(a.auth, r.Header.Get("Authorization"))
+	a.queries = append(a.queries, r.URL.RawQuery)
+	if raw, _ := io.ReadAll(r.Body); len(raw) > 0 {
+		a.bodies = append(a.bodies, string(raw))
+	}
+	q := a.replies[key]
+	if len(q) == 0 {
+		a.t.Errorf("unexpected call %s", key)
+		w.WriteHeader(http.StatusTeapot)
+		return
+	}
+	next := q[0]
+	if len(q) > 1 {
+		a.replies[key] = q[1:]
+	}
+	w.WriteHeader(next.status)
+	_, _ = io.WriteString(w, next.body)
+}
+
+func TestList_FiltersByProjectAndSendsTheToken(t *testing.T) {
+	a, c := newClient(t, map[string][]reply{"GET /elastic_ips": {{200, `{"data":[` +
+		eip("eip_000001", "203.0.113.1", "active", "sv_000001") + `,` + eip("eip_000002", "203.0.113.2", "active", "") + `]}`}}})
+
+	ips, err := c.ListElasticIPs(t.Context())
+	require.NoError(t, err)
+	require.Len(t, ips, 2)
+	assert.Equal(t, "sv_000001", ips[0].ServerID)
+	assert.Empty(t, ips[1].ServerID)
+	assert.Equal(t, []string{"Bearer " + testToken}, a.auth)
+	assert.Contains(t, a.queries[0], "filter%5Bproject%5D=proj_1")
+}
+
+func TestList_AFullPageIsAnError(t *testing.T) {
+	items := make([]string, listPageSize)
+	for i := range items {
+		items[i] = eip(fmt.Sprintf("eip_%06d", i), fmt.Sprintf("203.0.113.%d", i%250+1), "active", "")
+	}
+	_, c := newClient(t, map[string][]reply{"GET /elastic_ips": {{200, `{"data":[` + strings.Join(items, ",") + `]}`}}})
+	_, err := c.ListElasticIPs(t.Context())
+	assert.ErrorContains(t, err, "possibly truncated")
+}
+
+func TestGet_NullIDIsRejected(t *testing.T) {
+	_, c := newClient(t, map[string][]reply{"GET /elastic_ips/eip_000001": {{200, one("", "203.0.113.1", "configuring", "")}}})
+	_, err := c.GetElasticIP(t.Context(), "eip_000001")
+	assert.ErrorContains(t, err, "no id")
+}
+
+func TestGet_NotFound(t *testing.T) {
+	_, c := newClient(t, map[string][]reply{"GET /elastic_ips/eip_000001": {{404, errBody("NOT_FOUND")}}})
+	_, err := c.GetElasticIP(t.Context(), "eip_000001")
+	assert.ErrorIs(t, err, ErrNotFound)
+}
+
+func TestMove_PatchesThenWaitsForActive(t *testing.T) {
+	a, c := newClient(t, map[string][]reply{
+		"GET /elastic_ips/eip_000001": {
+			{200, one("eip_000001", "203.0.113.1", "active", "sv_000001")},
+			{200, one("eip_000001", "203.0.113.1", "moving", "sv_000001")},
+			{200, one("eip_000001", "203.0.113.1", "active", "sv_000002")},
+		},
+		"PATCH /elastic_ips/eip_000001": {{202, one("eip_000001", "203.0.113.1", "moving", "sv_000001")}},
+	})
+	require.NoError(t, c.MoveElasticIP(t.Context(), "eip_000001", "sv_000002"))
+
+	require.Len(t, a.bodies, 1)
+	var body struct {
+		Data struct {
+			Type       string            `json:"type"`
+			Attributes map[string]string `json:"attributes"`
+		} `json:"data"`
+	}
+	require.NoError(t, json.Unmarshal([]byte(a.bodies[0]), &body))
+	assert.Equal(t, "elastic_ips", body.Data.Type)
+	assert.Equal(t, map[string]string{"server_id": "sv_000002"}, body.Data.Attributes)
+}
+
+func TestMove_AlreadyThereMakesNoPatch(t *testing.T) {
+	_, c := newClient(t, map[string][]reply{
+		"GET /elastic_ips/eip_000001": {{200, one("eip_000001", "203.0.113.1", "active", "sv_000002")}},
+	})
+	require.NoError(t, c.MoveElasticIP(t.Context(), "eip_000001", "sv_000002"))
+}
+
+// The caller checked ownership just before calling, so a call must never wait and then write.
+func TestMove_NotActiveStateMakesNoWrite(t *testing.T) {
+	a, c := newClient(t, map[string][]reply{
+		"GET /elastic_ips/eip_000001": {{200, one("eip_000001", "203.0.113.1", "moving", "sv_000001")}},
+	})
+	err := c.MoveElasticIP(t.Context(), "eip_000001", "sv_000002")
+	assert.ErrorIs(t, err, ErrNotActive)
+	assert.Empty(t, a.bodies, "no PATCH was sent")
+}
+
+func TestMove_RefusedAsNotActiveIsNotRetried(t *testing.T) {
+	a, c := newClient(t, map[string][]reply{
+		"GET /elastic_ips/eip_000001":   {{200, one("eip_000001", "203.0.113.1", "active", "sv_000001")}},
+		"PATCH /elastic_ips/eip_000001": {{422, errBody("ELASTIC_IP_NOT_ACTIVE")}},
+	})
+	err := c.MoveElasticIP(t.Context(), "eip_000001", "sv_000002")
+	assert.ErrorIs(t, err, ErrNotActive)
+	assert.Len(t, a.bodies, 1, "exactly one PATCH")
+}
+
+func TestMove_NeverIssuesASecondWriteWhileWaiting(t *testing.T) {
+	a, c := newClient(t, map[string][]reply{
+		"GET /elastic_ips/eip_000001": {
+			{200, one("eip_000001", "203.0.113.1", "active", "sv_000001")},
+			{200, one("eip_000001", "203.0.113.1", "moving", "sv_000001")},
+			{200, one("eip_000001", "203.0.113.1", "active", "sv_000001")}, // another node moved it back meanwhile
+			{200, one("eip_000001", "203.0.113.1", "active", "sv_000002")},
+		},
+		"PATCH /elastic_ips/eip_000001": {{202, one("eip_000001", "203.0.113.1", "moving", "sv_000001")}},
+	})
+	require.NoError(t, c.MoveElasticIP(t.Context(), "eip_000001", "sv_000002"))
+	assert.Len(t, a.bodies, 1, "waiting only reads")
+}
+
+func TestMove_BGPAddressIsRefusedWithoutAWrite(t *testing.T) {
+	a, c := newClient(t, map[string][]reply{
+		"GET /elastic_ips/eip_bbbbbb": {{200, `{"data":` + bgp("eip_bbbbbb", "203.0.113.9") + `}`}},
+	})
+	err := c.MoveElasticIP(t.Context(), "eip_bbbbbb", "sv_000002")
+	assert.ErrorIs(t, err, ErrMoveRefused)
+	assert.Empty(t, a.bodies)
+}
+
+func TestUnusable(t *testing.T) {
+	ok := ElasticIP{Mode: "routed", Family: "IPv4", PrefixLength: 32}
+	assert.Empty(t, ok.Unusable())
+	for name, e := range map[string]ElasticIP{
+		"bgp":    {Mode: "bgp", Family: "IPv4", PrefixLength: 32},
+		"v6":     {Mode: "routed", Family: "IPv6", PrefixLength: 32},
+		"prefix": {Mode: "routed", Family: "IPv4", PrefixLength: 24},
+		"empty":  {},
+	} {
+		assert.NotEmpty(t, e.Unusable(), name)
+	}
+}
+
+func TestErrors_NeverCarryTheBodyOrTheToken(t *testing.T) {
+	bodies := map[string]string{
+		"json detail": `{"errors":[{"code":"BAD_THING","title":"x","detail":"echo ` + testToken + `"}]}`,
+		"json code":   `{"errors":[{"code":"` + testToken + `","detail":"y"}]}`,
+		"plain text":  "Authorization: Bearer " + testToken,
+		"html":        "<html>" + testToken + "</html>",
+	}
+	for name, body := range bodies {
+		for _, status := range []int{400, 404, 422, 401} {
+			t.Run(fmt.Sprintf("%s %d", name, status), func(t *testing.T) {
+				_, c := newClient(t, map[string][]reply{"GET /elastic_ips/eip_000001": {{status, body}}})
+				_, err := c.GetElasticIP(t.Context(), "eip_000001")
+				require.Error(t, err)
+				assert.NotContains(t, err.Error(), testToken)
+				assert.NotContains(t, fmt.Sprintf("%+v", err), testToken)
+			})
+		}
+	}
+}
+
+func TestErrors_KeepTheStatusAndAKnownCode(t *testing.T) {
+	_, c := newClient(t, map[string][]reply{"GET /elastic_ips/eip_000001": {{422, errBody("ELASTIC_IP_NOT_ACTIVE")}}})
+	_, err := c.GetElasticIP(t.Context(), "eip_000001")
+	assert.EqualError(t, err, "latitude: HTTP 422 ELASTIC_IP_NOT_ACTIVE")
+}
+
+func TestErrors_ADecodeFailureDoesNotEchoTheBody(t *testing.T) {
+	_, c := newClient(t, map[string][]reply{"GET /elastic_ips/eip_000001": {{200, `{"data": ` + testToken}}})
+	_, err := c.GetElasticIP(t.Context(), "eip_000001")
+	require.Error(t, err)
+	assert.NotContains(t, err.Error(), testToken)
+}
+
+func TestMove_RefusalsAreFinal(t *testing.T) {
+	for _, code := range []string{"SERVER_NOT_IN_PROJECT", "SITE_MISMATCH", "SERVER_NETWORK_INCOMPATIBLE"} {
+		t.Run(code, func(t *testing.T) {
+			_, c := newClient(t, map[string][]reply{
+				"GET /elastic_ips/eip_000001":   {{200, one("eip_000001", "203.0.113.1", "active", "sv_000001")}},
+				"PATCH /elastic_ips/eip_000001": {{422, errBody(code)}},
+			})
+			err := c.MoveElasticIP(t.Context(), "eip_000001", "sv_000002")
+			assert.ErrorIs(t, err, ErrMoveRefused)
+			assert.ErrorContains(t, err, code)
+		})
+	}
+}
+
+func TestMove_ErrorStateIsRefused(t *testing.T) {
+	_, c := newClient(t, map[string][]reply{"GET /elastic_ips/eip_000001": {{200, one("eip_000001", "203.0.113.1", "error", "")}}})
+	assert.ErrorIs(t, c.MoveElasticIP(t.Context(), "eip_000001", "sv_000002"), ErrMoveRefused)
+}
+
+func TestMove_NeverSettlingTimesOut(t *testing.T) {
+	_, c := newClient(t, map[string][]reply{
+		"GET /elastic_ips/eip_000001": {
+			{200, one("eip_000001", "203.0.113.1", "active", "sv_000001")},
+			{200, one("eip_000001", "203.0.113.1", "moving", "sv_000001")},
+		},
+		"PATCH /elastic_ips/eip_000001": {{202, one("eip_000001", "203.0.113.1", "moving", "sv_000001")}},
+	})
+	c.cfg.MoveTimeout = 30 * time.Millisecond
+	assert.ErrorIs(t, c.MoveElasticIP(t.Context(), "eip_000001", "sv_000002"), ErrNotSettled)
+}
+
+func TestDo_RetriesAThrottleThenSucceeds(t *testing.T) {
+	_, c := newClient(t, map[string][]reply{"GET /elastic_ips/eip_000001": {
+		{429, `{"meta":{"retry_after":1}}`},
+		{200, one("eip_000001", "203.0.113.1", "active", "")},
+	}})
+	ip, err := c.GetElasticIP(t.Context(), "eip_000001")
+	require.NoError(t, err)
+	assert.Equal(t, "eip_000001", ip.ID)
+}
+
+func TestDo_RefusedTokenIsUnauthorized(t *testing.T) {
+	_, c := newClient(t, map[string][]reply{"GET /elastic_ips": {{401, errBody("UNAUTHORIZED")}}})
+	_, err := c.ListElasticIPs(t.Context())
+	assert.ErrorIs(t, err, ErrUnauthorized)
+}
+
+func TestNew_NeedsAProject(t *testing.T) {
+	_, err := New(Config{})
+	assert.Error(t, err)
+}
+
+func TestNew_EmptyTokenFileFailsTheCall(t *testing.T) {
+	f := filepath.Join(t.TempDir(), "token")
+	require.NoError(t, os.WriteFile(f, []byte("\n"), 0o600))
+	c, err := New(Config{KeyFile: f, ProjectID: "p", BaseURL: "http://127.0.0.1:1"})
+	require.NoError(t, err)
+	_, err = c.ListElasticIPs(t.Context())
+	assert.ErrorContains(t, err, "empty")
+}
+
+// A write is one attempt: a throttle, a server failure or a lost connection after the
+// caller's ownership check must not turn into a second PATCH.
+func TestMove_PatchIsExactlyOneAttempt(t *testing.T) {
+	for _, status := range []int{429, 500, 502, 503} {
+		t.Run(strconv.Itoa(status), func(t *testing.T) {
+			a, c := newClient(t, map[string][]reply{
+				"GET /elastic_ips/eip_000001":   {{200, one("eip_000001", "203.0.113.1", "active", "sv_000001")}},
+				"PATCH /elastic_ips/eip_000001": {{status, `{}`}},
+			})
+			err := c.MoveElasticIP(t.Context(), "eip_000001", "sv_000002")
+			assert.ErrorIs(t, err, ErrUnavailable)
+			assert.Len(t, a.bodies, 1, "exactly one PATCH")
+		})
+	}
+}
+
+func TestMove_PatchTransportFailureIsExactlyOneAttempt(t *testing.T) {
+	var mu sync.Mutex
+	patches := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodGet {
+			_, _ = io.WriteString(w, one("eip_000001", "203.0.113.1", "active", "sv_000001"))
+			return
+		}
+		mu.Lock()
+		patches++
+		mu.Unlock()
+		if conn, _, err := w.(http.Hijacker).Hijack(); err == nil {
+			_ = conn.Close()
+		}
+	}))
+	t.Cleanup(srv.Close)
+	keyFile := filepath.Join(t.TempDir(), "token")
+	require.NoError(t, os.WriteFile(keyFile, []byte(testToken), 0o600))
+	c, err := New(Config{BaseURL: srv.URL, KeyFile: keyFile, ProjectID: "p", Backoff: time.Millisecond, PollInterval: time.Millisecond})
+	require.NoError(t, err)
+
+	err = c.MoveElasticIP(t.Context(), "eip_000001", "sv_000002")
+	assert.ErrorIs(t, err, ErrUnavailable)
+	mu.Lock()
+	defer mu.Unlock()
+	assert.Equal(t, 1, patches)
+}
+
+func TestErrors_OnlyDocumentedCodesAreShown(t *testing.T) {
+	_, c := newClient(t, map[string][]reply{"GET /elastic_ips/eip_000001": {{422, errBody(testToken)}}})
+	_, err := c.GetElasticIP(t.Context(), "eip_000001")
+	require.Error(t, err)
+	assert.NotContains(t, err.Error(), testToken)
+	assert.EqualError(t, err, "latitude: HTTP 422 Unprocessable Entity")
+}
+
+// A 200 can carry the token too, in a field the client reads.
+func TestErrors_SuccessResponseFieldsNeverReachAnError(t *testing.T) {
+	badAddr := `{"data":{"id":"eip_000001","attributes":{"address":"` + testToken + `","status":"active","mode":"routed","family":"IPv4","prefix_length":32}}}`
+	badList := `{"data":[{"id":"` + testToken + `","attributes":{"address":"` + testToken + `"}}]}`
+	_, c := newClient(t, map[string][]reply{
+		"GET /elastic_ips/eip_000001": {{200, badAddr}},
+		"GET /elastic_ips":            {{200, badList}},
+	})
+	_, err := c.GetElasticIP(t.Context(), "eip_000001")
+	require.Error(t, err)
+	assert.NotContains(t, err.Error(), testToken)
+	_, err = c.ListElasticIPs(t.Context())
+	require.Error(t, err)
+	assert.NotContains(t, err.Error(), testToken)
+}
+
+func TestErrors_ResponseEnumsAreShownOnlyWhenDocumented(t *testing.T) {
+	odd := func(status, mode string) string {
+		return `{"data":{"id":"eip_000001","attributes":{"address":"203.0.113.1","status":"` + status + `","mode":"` + mode + `","family":"` + testToken + `","prefix_length":32,"server":null}}}`
+	}
+	_, c := newClient(t, map[string][]reply{"GET /elastic_ips/eip_000001": {{200, odd("active", testToken)}}})
+	err := c.MoveElasticIP(t.Context(), "eip_000001", "sv_000002")
+	require.ErrorIs(t, err, ErrMoveRefused)
+	assert.NotContains(t, err.Error(), testToken)
+
+	_, c = newClient(t, map[string][]reply{"GET /elastic_ips/eip_000001": {{200, `{"data":{"id":"eip_000001","attributes":{"address":"203.0.113.1","status":"` + testToken + `","mode":"routed","family":"IPv4","prefix_length":32}}}`}}})
+	err = c.MoveElasticIP(t.Context(), "eip_000001", "sv_000002")
+	require.ErrorIs(t, err, ErrNotActive)
+	assert.NotContains(t, err.Error(), testToken)
+}
+
+// A 307 or 308 would replay the PATCH body to wherever it points: a second write.
+func TestMove_RedirectsAreNotFollowed(t *testing.T) {
+	for _, status := range []int{http.StatusTemporaryRedirect, http.StatusPermanentRedirect} {
+		t.Run(strconv.Itoa(status), func(t *testing.T) {
+			var mu sync.Mutex
+			patches := 0
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.Method == http.MethodGet {
+					_, _ = io.WriteString(w, one("eip_000001", "203.0.113.1", "active", "sv_000001"))
+					return
+				}
+				mu.Lock()
+				patches++
+				mu.Unlock()
+				w.Header().Set("Location", r.URL.Path)
+				w.WriteHeader(status)
+			}))
+			t.Cleanup(srv.Close)
+			keyFile := filepath.Join(t.TempDir(), "token")
+			require.NoError(t, os.WriteFile(keyFile, []byte(testToken), 0o600))
+			c, err := New(Config{BaseURL: srv.URL, KeyFile: keyFile, ProjectID: "p", HTTPClient: srv.Client()})
+			require.NoError(t, err)
+
+			require.Error(t, c.MoveElasticIP(t.Context(), "eip_000001", "sv_000002"))
+			mu.Lock()
+			defer mu.Unlock()
+			assert.Equal(t, 1, patches)
+		})
+	}
+}
+
+// IDs are opaque: whatever the API sends is returned unchanged, so equality against a
+// configured ID stays exact. Short IDs and ones that look like the token are not rewritten.
+func TestIDs_ReadFromAResponseAreReturnedUnchanged(t *testing.T) {
+	for _, holder := range []string{"sv_a", testToken, "not an id"} {
+		odd := `{"data":{"id":"eip_1","attributes":{"address":"203.0.113.1","status":"active","mode":"routed","family":"IPv4","prefix_length":32,"server":{"id":"` + holder + `"}}}}`
+		_, c := newClient(t, map[string][]reply{"GET /elastic_ips/eip_1": {{200, odd}}})
+		ip, err := c.GetElasticIP(t.Context(), "eip_1")
+		require.NoError(t, err)
+		assert.Equal(t, holder, ip.ServerID)
+		assert.Equal(t, "eip_1", ip.ID)
+	}
+}
@@ -0,0 +1,108 @@
+package latitude
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"sync"
+)
+
+// Fake is an in-memory stand-in for Client. It holds a fixed set of addresses
+// and moves one to any known server at once. It mirrors the documented
+// behaviour, not a measured one: a move to the current holder succeeds, and
+// moving to an unknown server is refused.
+type Fake struct {
+	mu      sync.Mutex
+	ips     map[string]ElasticIP
+	servers map[string]struct{}
+
+	// Fail, when set for an operation name ("get", "list", "move"), makes that
+	// operation return the error once.
+	Fail map[string]error
+	// Calls records operation names in order.
+	Calls []string
+}
+
+// NewFake returns a fake whose addresses may move among servers.
+func NewFake(servers ...string) *Fake {
+	f := &Fake{ips: map[string]ElasticIP{}, servers: map[string]struct{}{}, Fail: map[string]error{}}
+	for _, s := range servers {
+		f.servers[s] = struct{}{}
+	}
+	return f
+}
+
+// Seed adds an address. An empty status means active.
+func (f *Fake) Seed(ip ElasticIP) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if ip.Status == "" {
+		ip.Status = StatusActive
+	}
+	if ip.Mode == "" && ip.Family == "" && ip.PrefixLength == 0 {
+		ip.Mode, ip.Family, ip.PrefixLength = "routed", "IPv4", 32
+	}
+	f.ips[ip.ID] = ip
+}
+
+// HolderOf reports which server holds id; empty when unassigned or missing.
+func (f *Fake) HolderOf(id string) string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.ips[id].ServerID
+}
+
+// fail must be called with f.mu held.
+func (f *Fake) fail(op string) error {
+	f.Calls = append(f.Calls, op)
+	if err, ok := f.Fail[op]; ok {
+		delete(f.Fail, op)
+		return err
+	}
+	return nil
+}
+
+func (f *Fake) GetElasticIP(_ context.Context, id string) (ElasticIP, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.fail("get"); err != nil {
+		return ElasticIP{}, err
+	}
+	ip, ok := f.ips[id]
+	if !ok {
+		return ElasticIP{}, &APIError{Status: 404, Message: "not found", kind: ErrNotFound}
+	}
+	return ip, nil
+}
+
+func (f *Fake) ListElasticIPs(context.Context) ([]ElasticIP, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.fail("list"); err != nil {
+		return nil, err
+	}
+	out := make([]ElasticIP, 0, len(f.ips))
+	for _, ip := range f.ips {
+		out = append(out, ip)
+	}
+	slices.SortFunc(out, func(a, b ElasticIP) int { return a.Address.Compare(b.Address) })
+	return out, nil
+}
+
+func (f *Fake) MoveElasticIP(_ context.Context, id, serverID string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if err := f.fail("move"); err != nil {
+		return err
+	}
+	ip, ok := f.ips[id]
+	if !ok {
+		return &APIError{Status: 404, Message: "not found", kind: ErrNotFound}
+	}
+	if _, ok := f.servers[serverID]; !ok {
+		return &APIError{Status: 422, Code: "SERVER_NOT_IN_PROJECT", Message: fmt.Sprintf("server %s", serverID), kind: ErrMoveRefused}
+	}
+	ip.ServerID = serverID
+	f.ips[id] = ip
+	return nil
+}
@@ -137,7 +137,7 @@ func RunBenchSuite(b *testing.B, provider ebsprovider.EBSProvider, cfg BenchConf
 			}); err != nil {
 				b.Fatal(err)
 			}
-			if err := provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{
+			if _, err := provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{
 				Versioned: ebsprovider.NewVersioned(),
 				VolumeID:  volumeID,
 				NodeID:    cfg.node(),
--- a/Show More
+++ b/Show More