fix(latitude): stage and verify the CA before replacing it, treat a fresh pid-less lock as live

* docs(vultr): say where to run the cluster setup commands on Vultr

* docs(vultr): set up the cluster from node 1

cmd/lambda-agent/package_testlink.go

@@ -0,0 +1,276 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// Dev-only code delivery for the test path: the host attaches the ZIP as a
+// read-only disk and passes its exact size and SHA-256 through fw_cfg. This
+// is not the production delivery, and these bounds are not AWS limits.
+const (
+	entryPackageSize   = "test-package-size"
+	entryPackageSHA256 = "test-package-sha256"
+
+	devMaxZipBytes    = 16 << 20
+	devMaxUnzipBytes  = 64 << 20
+	devMaxEntries     = 1024
+	packageDevice     = "/dev/vda"
+	taskTmpfsSizeOpts = "mode=0755,size=72m"
+)
+
+type packageSpec struct {
+	size   int64
+	sha256 [sha256.Size]byte
+}
+
+type unpackLimits struct {
+	entries int
+	bytes   int64
+}
+
+var devLimits = unpackLimits{entries: devMaxEntries, bytes: devMaxUnzipBytes}
+
+// readPackageSpec reads the expected size and SHA-256 from fw_cfg; it must
+// run before fw_cfg is sealed.
+func readPackageSpec(fwCfgDir string) (packageSpec, error) {
+	var spec packageSpec
+	read := func(name string) (string, error) {
+		b, err := os.ReadFile(filepath.Join(fwCfgDir, name, "raw"))
+		if err != nil {
+			return "", fmt.Errorf("read %s: %w", name, err)
+		}
+		return strings.TrimSpace(string(b)), nil
+	}
+	s, err := read(entryPackageSize)
+	if err != nil {
+		return spec, err
+	}
+	spec.size, err = strconv.ParseInt(s, 10, 64)
+	if err != nil || spec.size <= 0 || spec.size > devMaxZipBytes {
+		return spec, fmt.Errorf("package size %q outside 1..%d bytes", s, devMaxZipBytes)
+	}
+	h, err := read(entryPackageSHA256)
+	if err != nil {
+		return spec, err
+	}
+	b, err := hex.DecodeString(h)
+	if err != nil || len(b) != sha256.Size {
+		return spec, errors.New("package sha256 is not 64 hex digits")
+	}
+	copy(spec.sha256[:], b)
+	return spec, nil
+}
+
+// taskPackage puts the package under dir: a fresh nosuid,nodev tmpfs, made
+// read-only once the files are in place.
+type taskPackage struct {
+	device     string
+	dir        string
+	loadDriver func() error
+	mountDir   func(dir string) error
+	sealDir    func(dir string) error
+}
+
+func devTaskPackage() taskPackage {
+	return taskPackage{device: packageDevice, dir: taskRoot, loadDriver: loadBlockDriver, mountDir: mountTaskDir, sealDir: sealTaskDir}
+}
+
+func (p taskPackage) install(spec packageSpec) error {
+	if err := p.loadDriver(); err != nil {
+		return fmt.Errorf("load block driver: %w", err)
+	}
+	dev, err := os.Open(p.device)
+	if err != nil {
+		return fmt.Errorf("open package disk: %w", err)
+	}
+	defer dev.Close()
+	if err := p.mountDir(p.dir); err != nil {
+		return fmt.Errorf("mount %s: %w", p.dir, err)
+	}
+	if err := unpack(dev, spec, p.dir, devLimits); err != nil {
+		return err
+	}
+	if err := p.sealDir(p.dir); err != nil {
+		return fmt.Errorf("make %s read-only: %w", p.dir, err)
+	}
+	return nil
+}
+
+// unpack reads exactly spec.size bytes once, checks their SHA-256, then
+// validates every entry before writing any of them under dir.
+func unpack(r io.ReaderAt, spec packageSpec, dir string, lim unpackLimits) error {
+	buf := make([]byte, spec.size)
+	if _, err := io.ReadFull(io.NewSectionReader(r, 0, spec.size), buf); err != nil {
+		return fmt.Errorf("read package: %w", err)
+	}
+	if sha256.Sum256(buf) != spec.sha256 {
+		return errors.New("package sha256 mismatch")
+	}
+	zr, err := zip.NewReader(bytes.NewReader(buf), spec.size)
+	if err != nil {
+		return fmt.Errorf("open package: %w", err)
+	}
+	if len(zr.File) > lim.entries {
+		return fmt.Errorf("package has %d entries, limit %d", len(zr.File), lim.entries)
+	}
+	names := make([]string, len(zr.File))
+	var declared int64
+	for i, f := range zr.File {
+		name, err := entryName(f.Name)
+		if err != nil {
+			return err
+		}
+		names[i] = name
+		switch mode := f.Mode(); {
+		case mode.IsDir():
+		case mode.IsRegular():
+			// Compared against what remains, so a huge size cannot wrap the sum.
+			if f.UncompressedSize64 > uint64(lim.bytes-declared) {
+				return fmt.Errorf("%q: unpacked size over the %d-byte limit", f.Name, lim.bytes)
+			}
+			declared += int64(f.UncompressedSize64)
+		default:
+			return fmt.Errorf("%q: %s entries are refused", f.Name, mode.Type())
+		}
+	}
+
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		return err
+	}
+	defer root.Close()
+	var written int64
+	for i, f := range zr.File {
+		n, err := extract(root, names[i], f, lim.bytes-written)
+		written += n
+		if err != nil {
+			return fmt.Errorf("%q: %w", f.Name, err)
+		}
+	}
+	return nil
+}
+
+// extract writes one entry, refusing it if its actual bytes exceed budget
+// whatever its header claims.
+func extract(root *os.Root, name string, f *zip.File, budget int64) (int64, error) {
+	if f.Mode().IsDir() {
+		return 0, mkdirs(root, name)
+	}
+	if d := path.Dir(name); d != "." {
+		if err := mkdirs(root, d); err != nil {
+			return 0, err
+		}
+	}
+	perm := os.FileMode(0o644)
+	if f.Mode()&0o111 != 0 {
+		perm = 0o755
+	}
+	in, err := f.Open()
+	if err != nil {
+		return 0, err
+	}
+	defer in.Close()
+	out, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
+	if err != nil {
+		return 0, err
+	}
+	// The umask applies at create; fchmod does not, so the mode is exact.
+	if err := out.Chmod(perm); err != nil {
+		out.Close()
+		return 0, err
+	}
+	n, err := io.Copy(out, io.LimitReader(in, budget+1))
+	if cerr := out.Close(); err == nil {
+		err = cerr
+	}
+	if err == nil && n > budget {
+		err = errors.New("unpacked bytes over the limit")
+	}
+	return n, err
+}
+
+// mkdirs creates each component of dir as mode 0755 whatever the umask. An
+// existing component must be a real directory, never a symlink.
+func mkdirs(root *os.Root, dir string) error {
+	cur := ""
+	for _, part := range strings.Split(dir, "/") {
+		cur = path.Join(cur, part)
+		if err := root.Mkdir(cur, 0o755); err != nil {
+			if !errors.Is(err, fs.ErrExist) {
+				return err
+			}
+			fi, err := root.Lstat(cur)
+			if err != nil {
+				return err
+			}
+			if !fi.IsDir() {
+				return fmt.Errorf("%q exists and is not a directory", cur)
+			}
+		}
+		if err := root.Chmod(cur, 0o755); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// entryName returns a clean relative path, refusing anything that could name
+// a location outside the task root.
+func entryName(name string) (string, error) {
+	if name == "" || strings.ContainsAny(name, "\\\x00") || path.IsAbs(name) {
+		return "", fmt.Errorf("%q: entry name refused", name)
+	}
+	c := path.Clean(name)
+	if c == "." || c == ".." || strings.HasPrefix(c, "../") {
+		return "", fmt.Errorf("%q: entry name refused", name)
+	}
+	return c, nil
+}
+
+// loadBlockDriver loads virtio_blk, which the image ships but init does not
+// load, then waits briefly for the package disk to appear.
+func loadBlockDriver() error {
+	if err := exec.Command("/sbin/modprobe", "virtio_blk").Run(); err != nil {
+		var uts unix.Utsname
+		if uerr := unix.Uname(&uts); uerr != nil {
+			return uerr
+		}
+		ko := "/lib/modules/" + unix.ByteSliceToString(uts.Release[:]) + "/kernel/drivers/block/virtio_blk.ko"
+		if err := exec.Command("/sbin/insmod", ko).Run(); err != nil {
+			return fmt.Errorf("modprobe and insmod virtio_blk failed: %w", err)
+		}
+	}
+	for range 50 {
+		if _, err := os.Stat(packageDevice); err == nil {
+			return nil
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	return fmt.Errorf("%s did not appear", packageDevice)
+}
+
+func mountTaskDir(dir string) error {
+	return unix.Mount("tmpfs", dir, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, taskTmpfsSizeOpts)
+}
+
+func sealTaskDir(dir string) error {
+	return unix.Mount("", dir, "", unix.MS_REMOUNT|unix.MS_RDONLY|unix.MS_NOSUID|unix.MS_NODEV, "")
+}

cmd/lambda-agent/package_testlink_test.go

@@ -0,0 +1,209 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"hash/crc32"
+	"io/fs"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testZip builds a ZIP; an entry named bootstrap is marked executable.
+func testZip(t *testing.T, files map[string]string) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	names := make([]string, 0, len(files))
+	for n := range files {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	for _, n := range names {
+		h := &zip.FileHeader{Name: n, Method: zip.Deflate}
+		h.SetMode(0o644)
+		if n == "bootstrap" {
+			h.SetMode(0o755)
+		}
+		w, err := zw.CreateHeader(h)
+		require.NoError(t, err)
+		_, err = w.Write([]byte(files[n]))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+// rawZip writes entries with headers set by the caller, so a test can claim
+// sizes or modes the data does not have.
+func rawZip(t *testing.T, entries ...*zip.FileHeader) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for _, h := range entries {
+		w, err := zw.CreateRaw(h)
+		require.NoError(t, err)
+		_, err = w.Write([]byte("x"))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+func specFor(b []byte) packageSpec {
+	return packageSpec{size: int64(len(b)), sha256: sha256.Sum256(b)}
+}
+
+// Not parallel: the umask is process-wide, so the test sets a restrictive one
+// and restores it.
+func TestUnpackInstallsFilesWithFixedModes(t *testing.T) {
+	old := syscall.Umask(0o077)
+	t.Cleanup(func() { syscall.Umask(old) })
+	b := testZip(t, map[string]string{"bootstrap": "#!/bin/sh\n", "lib/data.txt": "data"})
+	dir := t.TempDir()
+	require.NoError(t, unpack(bytes.NewReader(b), specFor(b), dir, devLimits))
+
+	pfi, err := os.Stat(filepath.Join(dir, "lib"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o755), pfi.Mode().Perm(), "implicit parent dir")
+
+	fi, err := os.Stat(filepath.Join(dir, "bootstrap"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o755), fi.Mode().Perm(), "executable")
+	fi, err = os.Stat(filepath.Join(dir, "lib", "data.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o644), fi.Mode().Perm())
+	got, err := os.ReadFile(filepath.Join(dir, "lib", "data.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, "data", string(got))
+}
+
+func TestUnpackRefusesBeforeWritingAnything(t *testing.T) {
+	symlink := &zip.FileHeader{Name: "link", Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: 1}
+	symlink.SetMode(fs.ModeSymlink | 0o777)
+	fifo := &zip.FileHeader{Name: "pipe", Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: 1}
+	fifo.SetMode(fs.ModeNamedPipe | 0o644)
+	// The second size would wrap a plain running sum back under the limit.
+	big := func(name string, n uint64) *zip.FileHeader {
+		return &zip.FileHeader{Name: name, Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: n}
+	}
+	cases := map[string]struct {
+		zip  []byte
+		want string
+	}{
+		"parent traversal": {testZip(t, map[string]string{"a": "ok", "../escape": "x"}), "entry name refused"},
+		"nested traversal": {testZip(t, map[string]string{"a": "ok", "lib/../../escape": "x"}), "entry name refused"},
+		"absolute path":    {testZip(t, map[string]string{"a": "ok", "/etc/escape": "x"}), "entry name refused"},
+		"backslash":        {testZip(t, map[string]string{"a": "ok", `..\escape`: "x"}), "entry name refused"},
+		"symlink":          {rawZip(t, big("a", 1), symlink), "entries are refused"},
+		"named pipe":       {rawZip(t, big("a", 1), fifo), "entries are refused"},
+		"declared size":    {rawZip(t, big("a", devMaxUnzipBytes+1)), "over the"},
+		"wrapping sizes":   {rawZip(t, big("a", devMaxUnzipBytes), big("b", math.MaxUint64)), "over the"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := t.TempDir()
+			err := unpack(bytes.NewReader(tc.zip), specFor(tc.zip), dir, devLimits)
+			assert.ErrorContains(t, err, tc.want)
+			left, rerr := os.ReadDir(dir)
+			require.NoError(t, rerr)
+			assert.Empty(t, left, "files written before the refusal")
+		})
+	}
+}
+
+func TestUnpackRefusesTooManyEntries(t *testing.T) {
+	b := testZip(t, map[string]string{"a": "1", "b": "2", "c": "3"})
+	err := unpack(bytes.NewReader(b), specFor(b), t.TempDir(), unpackLimits{entries: 2, bytes: devMaxUnzipBytes})
+	assert.ErrorContains(t, err, "limit 2")
+}
+
+// The header claims a size within the limit, but the data is longer: the
+// actual bytes are refused, whatever the header says.
+func TestUnpackRefusesBytesBeyondTheLimit(t *testing.T) {
+	data := bytes.Repeat([]byte("a"), 64)
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	h := &zip.FileHeader{Name: "a", Method: zip.Store, CRC32: crc32.ChecksumIEEE(data), CompressedSize64: uint64(len(data)), UncompressedSize64: 8}
+	w, err := zw.CreateRaw(h)
+	require.NoError(t, err)
+	_, err = w.Write(data)
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	b := buf.Bytes()
+
+	dir := t.TempDir()
+	err = unpack(bytes.NewReader(b), specFor(b), dir, unpackLimits{entries: 4, bytes: 16})
+	require.Error(t, err)
+	fi, serr := os.Stat(filepath.Join(dir, "a"))
+	if serr == nil {
+		assert.LessOrEqual(t, fi.Size(), int64(17))
+	}
+}
+
+func TestUnpackChecksSizeAndHash(t *testing.T) {
+	b := testZip(t, map[string]string{"bootstrap": "x"})
+	wrongHash := specFor(b)
+	wrongHash.sha256[0] ^= 1
+	assert.ErrorContains(t, unpack(bytes.NewReader(b), wrongHash, t.TempDir(), devLimits), "sha256 mismatch")
+
+	longer := specFor(b)
+	longer.size++
+	assert.ErrorContains(t, unpack(bytes.NewReader(b), longer, t.TempDir(), devLimits), "read package")
+}
+
+func TestReadPackageSpecBounds(t *testing.T) {
+	sum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	cases := map[string]struct {
+		size, hash string
+		want       string
+	}{
+		"valid":     {"10", sum, ""},
+		"zero":      {"0", sum, "outside"},
+		"too large": {"16777217", sum, "outside"},
+		"not hex":   {"10", "zz", "64 hex digits"},
+		"short":     {"10", sum[:62], "64 hex digits"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := t.TempDir()
+			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash} {
+				require.NoError(t, os.MkdirAll(filepath.Join(dir, n), 0o700))
+				require.NoError(t, os.WriteFile(filepath.Join(dir, n, "raw"), []byte(v), 0o600))
+			}
+			spec, err := readPackageSpec(dir)
+			if tc.want == "" {
+				require.NoError(t, err)
+				assert.Equal(t, int64(10), spec.size)
+				return
+			}
+			assert.ErrorContains(t, err, tc.want)
+		})
+	}
+}
+
+// A pre-existing symlink where a directory belongs is refused, not followed
+// or chmodded.
+func TestMkdirsRefusesSymlinkComponent(t *testing.T) {
+	dir := t.TempDir()
+	outside := t.TempDir()
+	require.NoError(t, os.Chmod(outside, 0o700))
+	require.NoError(t, os.Symlink(outside, filepath.Join(dir, "lib")))
+	root, err := os.OpenRoot(dir)
+	require.NoError(t, err)
+	defer root.Close()
+	assert.ErrorContains(t, mkdirs(root, "lib/sub"), "not a directory")
+	fi, err := os.Stat(outside)
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o700), fi.Mode().Perm(), "symlink target was chmodded")
+}

cmd/lambda-agent/start_testlink.go

@@ -30,6 +30,7 @@ var handler = flag.String("handler", os.Getenv("_HANDLER"), "function handler pa
 // connection; the carrier is not the guest protocol.
 type testlink struct {
 	startup
+	pkg            taskPackage
 	runtimeAddr    string
 	startBootstrap func(ctx context.Context, env []string) (*exec.Cmd, error)
 }
@@ -37,6 +38,7 @@ type testlink struct {
 func start(ctx context.Context) int {
 	t := testlink{
 		startup:     defaultStartup(),
+		pkg:         devTaskPackage(),
 		runtimeAddr: runtimeAddr,
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			return agent.StartBootstrap(ctx, taskRoot+"/bootstrap", env, os.Stdout, os.Stderr)
@@ -50,11 +52,16 @@ func start(ctx context.Context) int {
 }
 
 func (t testlink) run(parent context.Context) (int, error) {
+	// The package values are fw_cfg entries, so read them before connect seals it.
+	spec, specErr := readPackageSpec(t.fwCfgDir)
 	conn, code, err := t.connect(parent)
 	if err != nil {
-		return code, err
+		return code, errors.Join(err, specErr)
 	}
 	defer conn.Close()
+	if specErr != nil {
+		return exitRefused, specErr
+	}
 	link := hostlink.NewTestStream(conn)
 
 	// The handshake can return before the host refuses the guest, so nothing
@@ -69,6 +76,9 @@ func (t testlink) run(parent context.Context) (int, error) {
 	if parent.Err() != nil {
 		return exitLink, context.Cause(parent)
 	}
+	if err := t.pkg.install(spec); err != nil {
+		return exitRefused, fmt.Errorf("install package: %w", err)
+	}
 
 	srv := runtimeapi.New()
 	ln, err := net.Listen("tcp", t.runtimeAddr)

cmd/lambda-agent/start_testlink_test.go

@@ -5,10 +5,14 @@ package main
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"io"
 	"net/http"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"strconv"
 	"testing"
 	"time"
 
@@ -43,12 +47,31 @@ func TestHelperBootstrap(t *testing.T) {
 type bootstrapStarts struct {
 	n       int
 	started chan struct{}
+	task    string
+	sealed  bool
 }
 
-func (b *bootstrapStarts) testlink(g *guest) testlink {
+// testlink builds a guest whose fw_cfg also carries a controlled package,
+// installed into a temp dir without mounting anything.
+func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) testlink {
+	t.Helper()
+	zipBytes := testZip(t, map[string]string{"bootstrap": "#!/bin/sh\n", "lib/data.txt": "data"})
+	disk := filepath.Join(t.TempDir(), "disk")
+	require.NoError(t, os.WriteFile(disk, zipBytes, 0o600))
+	sum := sha256.Sum256(zipBytes)
+	entries[entryPackageSize] = []byte(strconv.Itoa(len(zipBytes)))
+	entries[entryPackageSHA256] = []byte(hex.EncodeToString(sum[:]))
+	g := newGuest(t, entries)
 	b.started = make(chan struct{}, 1)
+	b.task = t.TempDir()
 	return testlink{
-		startup:     g.startup(),
+		startup: g.startup(),
+		pkg: taskPackage{
+			device: disk, dir: b.task,
+			loadDriver: func() error { return nil },
+			mountDir:   func(string) error { return nil },
+			sealDir:    func(string) error { b.sealed = true; return nil },
+		},
 		runtimeAddr: "127.0.0.1:0",
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			b.n++
@@ -80,7 +103,7 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
 
 	conn := h.guest(t)
 	defer conn.Close()
@@ -109,6 +132,8 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		assert.Error(t, err, "agent answered a second invocation")
 	}
 	assert.Equal(t, 1, b.n)
+	assert.True(t, b.sealed, "task dir not made read-only")
+	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
 }
 
 func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
@@ -117,7 +142,7 @@ func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
 
 	conn := h.guest(t)
 	defer conn.Close()
@@ -142,7 +167,7 @@ func TestTestlinkHostRefusalNeverStartsBootstrap(t *testing.T) {
 	entries[lambdav1.EntryGuestKey], entries[lambdav1.EntryGuestCert] = wrong[lambdav1.EntryGuestKey], wrong[lambdav1.EntryGuestCert]
 	var b bootstrapStarts
 
-	res := wait(t, runTestlinkAsync(ctx, b.testlink(newGuest(t, entries))), "agent to stop")
+	res := wait(t, runTestlinkAsync(ctx, b.testlink(t, entries)), "agent to stop")
 	assert.Equal(t, exitLink, res.code)
 	require.Error(t, res.err)
 	assert.Contains(t, res.err.Error(), "bad certificate")
@@ -155,7 +180,7 @@ func TestTestlinkHostCloseBeforeFrameNeverStartsBootstrap(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
 
 	require.NoError(t, h.guest(t).Close())
 	res := wait(t, done, "agent to stop")
@@ -177,3 +202,45 @@ func TestOneInvocationHonoursCancelledContext(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "r-1", req.RequestID)
 }
+
+// A package that fails its check is refused after authentication, and the
+// bootstrap never starts.
+func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	tl := b.testlink(t, h.entries())
+	tl.pkg.device = filepath.Join(t.TempDir(), "other")
+	require.NoError(t, os.WriteFile(tl.pkg.device, testZip(t, map[string]string{"bootstrap": "x"}), 0o600))
+	done := runTestlinkAsync(ctx, tl)
+
+	conn := h.guest(t)
+	defer conn.Close()
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{RequestID: "r-1"}))
+	res := wait(t, done, "agent to stop")
+	assert.Equal(t, exitRefused, res.code)
+	assert.ErrorContains(t, res.err, "install package")
+	assert.Zero(t, b.n)
+	assert.False(t, b.sealed)
+}
+
+// Missing package values still seal fw_cfg, and the agent refuses before
+// waiting for any frame.
+func TestTestlinkMissingPackageSpecRefuses(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	tl := b.testlink(t, h.entries())
+	require.NoError(t, os.RemoveAll(filepath.Join(tl.fwCfgDir, entryPackageSHA256)))
+
+	res := wait(t, runTestlinkAsync(ctx, tl), "agent to stop")
+	assert.Equal(t, exitRefused, res.code)
+	assert.ErrorContains(t, res.err, "read "+entryPackageSHA256)
+	assert.Zero(t, b.n)
+	_, err := os.Stat(tl.fwCfgRoot)
+	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg not sealed")
+}

contracts/operator/v1/admin.go

@@ -25,6 +25,10 @@ const (
 	DeletionStateRunning   = "RUNNING"
 	DeletionStateCompleted = "COMPLETED"
 	DeletionStateFailed    = "FAILED"
+
+	// DeletionStateExercised ends a non-finalizing run: the account is left
+	// TERMINATING and a later DeleteAccount takes the same operation over.
+	DeletionStateExercised = "EXERCISED"
 )
 
 // CreateAccountRequest is the /admin/CreateAccount request body.
@@ -52,6 +56,10 @@ type DeleteAccountRequest struct {
 	ClientToken string `json:"clientToken"`
 	Force       bool   `json:"force,omitempty"`
 	DryRun      bool   `json:"dryRun,omitempty"`
+
+	// NonFinalizing quiesces the account, cleans the quiesced EC2 platform
+	// kinds and stops. No legacy stage runs and the account record stays.
+	NonFinalizing bool `json:"nonFinalizing,omitempty"`
 }
 
 // DescribeAccountDeletionRequest is the /admin/DescribeAccountDeletion body.
@@ -76,8 +84,8 @@ type ListAccountsResponse struct {
 }
 
 // IAMQuiescence is the coordinator's recorded view of IAM quiescence for one
-// fence. Recording only: it admits nothing, and QuiescedScope stays empty until
-// a final inventory re-read exists to catch a late write.
+// fence. Recording only: it admits nothing. QuiescedScope is filled only for a
+// non-finalizing run, and holds as of EvaluatedAt: a write after that is not caught.
 type IAMQuiescence struct {
 	OperationID   string            `json:"operationId"`
 	Fence         uint64            `json:"fence"`
@@ -136,3 +144,67 @@ const (
 	ReasonOwnerInventoryIncomplete     = "owner-inventory-incomplete"
 	ReasonFinalInventoryRereadNotWired = "final-inventory-reread-not-wired"
 )
+
+// ExerciseResult is what a non-finalizing run did after IAM quiescence.
+type ExerciseResult struct {
+	// Skipped is the fixed reason no cleanup ran; Cleanup is then absent.
+	Skipped    string           `json:"skipped,omitempty"`
+	Cleanup    *PlatformCleanup `json:"cleanup,omitempty"`
+	Accounting []KindAccounting `json:"accounting,omitempty"`
+}
+
+// PlatformCleanup is the EC2 platform cleanup result. It carries resource IDs,
+// fixed reasons and counts only, never names, tags, key material or data.
+type PlatformCleanup struct {
+	Kinds  []CleanupKind  `json:"kinds"`
+	Items  []CleanupItem  `json:"items,omitempty"`
+	Counts []CleanupCount `json:"counts,omitempty"`
+}
+
+// CleanupKind is one kind's scan status after the cleanup's own re-scan.
+type CleanupKind struct {
+	Kind              string   `json:"kind"`
+	Status            string   `json:"status"`
+	IncompleteReasons []string `json:"incompleteReasons,omitempty"`
+}
+
+// CleanupItem is one resource by its ID with what the cleanup did to it.
+type CleanupItem struct {
+	ResourceID  string `json:"resourceId"`
+	Kind        string `json:"kind"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	MemberCount int    `json:"memberCount,omitempty"`
+}
+
+// CleanupCount is the number of records of one class that have no public ID.
+type CleanupCount struct {
+	Kind        string `json:"kind"`
+	Class       string `json:"class"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	Count       int    `json:"count"`
+}
+
+// KindAccounting is the final inventory re-read held against the cleanup result
+// for one kind. Quiescence is not rewritten from it.
+type KindAccounting struct {
+	Kind      string `json:"kind"`
+	Status    string `json:"status"`
+	Reason    string `json:"reason,omitempty"`
+	Remaining int    `json:"remaining"`
+}
+
+// Accounting statuses and fixed reasons of a KindAccounting.
+const (
+	AccountingAccounted   = "accounted"
+	AccountingUnaccounted = "unaccounted"
+
+	AccountingReasonRereadIncomplete = "reread-incomplete"
+	AccountingReasonUnnamedRemainder = "unnamed-remainder"
+)
+
+// Fixed reasons an exercise ran no cleanup.
+const (
+	ExerciseSkippedNoQuiescedKind = "no-quiesced-kind"
+)

contracts/operator/v1/admin_test.go

@@ -43,6 +43,27 @@ func TestAdminWire_DeletionStates(t *testing.T) {
 	require.Equal(t, "RUNNING", DeletionStateRunning)
 	require.Equal(t, "COMPLETED", DeletionStateCompleted)
 	require.Equal(t, "FAILED", DeletionStateFailed)
+	require.Equal(t, "EXERCISED", DeletionStateExercised)
+}
+
+func TestAdminWire_NonFinalizingDeleteRequest(t *testing.T) {
+	goldenJSON(t, DeleteAccountRequest{AccountID: "000000000042", AccountName: "acme", ClientToken: "tok-3", NonFinalizing: true},
+		`{"accountId":"000000000042","accountName":"acme","clientToken":"tok-3","nonFinalizing":true}`)
+}
+
+func TestAdminWire_ExerciseResult(t *testing.T) {
+	goldenJSON(t, ExerciseResult{
+		Cleanup: &PlatformCleanup{
+			Kinds:  []CleanupKind{{Kind: "key-pair", Status: "observed-clean-at-scan-time"}},
+			Items:  []CleanupItem{{ResourceID: "pg-1", Kind: "placement-group", Disposition: "blocked", Reason: "instance-members", MemberCount: 1}},
+			Counts: []CleanupCount{{Kind: "key-pair", Class: "public-key-object-without-metadata", Disposition: "deleted", Count: 2}},
+		},
+		Accounting: []KindAccounting{{Kind: "placement-group", Status: AccountingAccounted, Remaining: 1}},
+	}, `{"cleanup":{"kinds":[{"kind":"key-pair","status":"observed-clean-at-scan-time"}],
+		"items":[{"resourceId":"pg-1","kind":"placement-group","disposition":"blocked","reason":"instance-members","memberCount":1}],
+		"counts":[{"kind":"key-pair","class":"public-key-object-without-metadata","disposition":"deleted","count":2}]},
+		"accounting":[{"kind":"placement-group","status":"accounted","remaining":1}]}`)
+	goldenJSON(t, ExerciseResult{Skipped: ExerciseSkippedNoQuiescedKind}, `{"skipped":"no-quiesced-kind"}`)
 }
 
 func TestAdminWire_ListAccountsResponse(t *testing.T) {

docs/DESIGN.md

@@ -65,9 +65,10 @@ The EC2 registration (`spinifex/domains/ec2/awsapi`, `NewRegistration`) parses t
 The gateway communicates with daemons via NATS request/response. Most calls go through `natsmsg.NATSRequest`, which marshals the input, attaches the account ID as a NATS header, and unmarshals the typed response:
 
 ```go
-func (s *NATSInstanceService) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
-    topic := fmt.Sprintf("ec2.RunInstances.%s", aws.StringValue(input.InstanceType))
-    return natsmsg.NATSRequest[ec2.Reservation](ctx, s.natsConn, topic, input, 5*time.Minute, accountID)
+func runIntoReservation(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, accountID, crID string) (*ec2.Reservation, error) {
+    subject := "ec2.RunInstances.cr." + crID
+    reservation, err := natsmsg.NATSRequest[ec2.Reservation](ctx, natsConn, subject, input, 5*time.Minute, accountID)
+    // ...
 }
 ```
 

docs/containers/eks/README.md

@@ -59,6 +59,9 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM
 
 - **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
 - **The cluster security group is auto-managed.** As on EKS, Spinifex creates one `eks-cluster-sg-<name>-<id>` in your VPC, tagged `kubernetes.io/cluster/<name>=owned` and `aws:eks:cluster-name`, allowing all traffic from itself. It is attached to the workers and the private-endpoint network interface, and `DescribeCluster` reports it as `clusterSecurityGroupId`. A launch template that names security groups replaces it on the workers, as on AWS. Clusters created before this change keep their `-control-plane-sg` and `-nodegroup-sg` until you recreate them. Deleting a cluster deletes the cluster security group and removes only the rules on your other security groups that reference it. `vpc_config.security_group_ids` is ignored.
+- **Node group deletes are asynchronous.** `DeleteNodegroup` returns at once with the group `DELETING`, and `DescribeNodegroup` answers `ResourceNotFoundException` once the workers and the group's instance profile are gone. Deleting a cluster or creating a group of the same name while a group is `DELETING` returns `ResourceInUseException`, as does a second delete of a `DELETING` group.
+- **If a node group delete cannot finish,** the group moves to `DELETE_FAILED` after 30 minutes, with a `NODE_TERMINATION_FAILURE` health issue naming the workers that are still running. Run `DeleteNodegroup` again to resume it.
+- **Each node group owns an EKS-managed instance profile.** Spinifex creates it, tags it `spinifex:eks-cluster`, `spinifex:eks-nodegroup` and `spinifex:eks-profile-id`, and puts the node role in it. It is removed with the group, so you can delete the node role afterwards with no extra steps. Spinifex never deletes an instance profile it did not create, or one whose tags, name or role no longer match what it recorded; it leaves the profile and logs a warning with its name.
 - **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
 - **Workers launch with IMDSv2 required and an IMDS hop limit of 2.**
   This matches AWS managed node groups and lets a pod without IRSA reach the node role through the SDK default credential chain.
@@ -377,6 +380,15 @@ Control plane readiness and kubectl authentication above lists which clusters re
 
 **`create-cluster` rejected with an authentication-mode error.** `CONFIG_MAP` is the one mode Spinifex rejects, because it has no `aws-auth` ConfigMap path. Set `authentication_mode` to `API` or `API_AND_CONFIG_MAP`, or leave it unset, and grant access with access entries.
 
+**`delete-role` fails with `DeleteConflict` after deleting a node group.** Node groups created before per-group instance profiles left an instance profile named after the role, which Spinifex no longer removes because it cannot prove it created it. Remove it yourself:
+
+```bash
+aws iam remove-role-from-instance-profile --instance-profile-name <role-name> --role-name <role-name>
+aws iam delete-instance-profile --instance-profile-name <role-name>
+```
+
+**A node group is `DELETE_FAILED`.** `aws eks describe-nodegroup` lists the workers that did not stop under `health.issues`. Once they are gone, or after you fix the cause, run `aws eks delete-nodegroup` again.
+
 **Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, use the AWS form from `DescribeAddonVersions` (see Add-on versions above), or leave `addon_version` unset so Spinifex selects the catalog default.
 
 ## Reference Architectures

docs/package-boundary/eks-cluster-lifecycle.md

@@ -34,7 +34,7 @@ Node group workers and CSI-provisioned EBS volumes are never removed by a cluste
 
 ## Gaps
 
-1. Four `eks.*` handlers still hold their request for work that grows with the resource, and the subject table names them as not conformant: `CreateCluster` (reclaiming a `FAILED` cluster of the same name tears its infra down inline), `DeleteNodegroup` (one awaited terminate per worker), `UpdateNodegroupConfig` (one awaited launch or terminate per worker added or removed) and `RestoreSnapshot` (launches a control-plane VM and fences the old one inline).
+1. Three `eks.*` handlers still hold their request for work that grows with the resource, and the subject table names them as not conformant: `CreateCluster` (reclaiming a `FAILED` cluster of the same name tears its infra down inline), `UpdateNodegroupConfig` (one awaited launch or terminate per worker added or removed) and `RestoreSnapshot` (launches a control-plane VM and fences the old one inline).
 2. AWS's answer to a repeat delete on a `DELETING` cluster has not been observed; `DELETING` from the record is the chosen behaviour.
 3. Control-plane, load balancer and VPC resources in the system account are keyed by the bare cluster name, so a teardown by name can reach another tenant's same-named cluster.
 4. The teardown deletes load balancers the in-cluster controller created in the customer account, which AWS leaves behind.

docs/vultr-integration/README.md

@@ -265,7 +265,12 @@ If this returns a list of available instance types, your installation is working
 
 Spinifex is running, but it holds nothing yet: no machine images, no networks, no instances.
 
-Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance.
+Log in to node 1, and follow [Setting Up Your Cluster](/docs/setting-up-your-cluster) there to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance:
+
+```bash
+./deploy-vultr.sh ssh
+```
+
 An instance you launch into a public subnet gets a Vultr reserved IP, which you can see in the Vultr portal under **Network → Reserved IPs**.
 
 ## 8. Test resiliency

scripts/terraform/latitude-spx/.gitignore

@@ -0,0 +1,10 @@
+.terraform/
+terraform.tfstate
+terraform.tfstate.*
+*.tfplan
+*.auto.tfvars
+crash.log
+terraform.tfvars
+*.pem
+*.log
+.terraform.lock.hcl

scripts/terraform/latitude-spx/README.md

@@ -0,0 +1,70 @@
+# latitude-spx
+
+Builds a one-node Spinifex deployment on latitude.sh bare metal, from your laptop, with `deploy-latitude.sh`. The shape follows `vultr-spx`.
+
+**Status: not yet run end to end.** The script was written from the plan doc findings (`docs/development/feature/latitude-bm.md`, L1 to L18) and from the provider schema. No latitude.sh API call or host run was made while building it. Treat the first `up` as the test.
+
+## What it creates
+
+| Resource | Notes |
+|---|---|
+| Project | `spinifex-<name>`, unless `--project` names an existing one (then it is never deleted) |
+| SSH key | `spinifex-<name>`, account scoped |
+| User data | `spinifex-<name>`, host preparation only (no `{{ }}`, latitude.sh substitutes them) |
+| Server | `m4-metal-medium`, site `DAL`, Ubuntu 26.04, **hourly** billing, hostname `spx-<name>-node1` |
+| Elastic IP | one, routed to node 1; it is the Spinifex public pool address |
+
+## Requirements
+
+OpenTofu (or Terraform) 1.5 or later, `aws` CLI, `python3`, `curl`, `ssh`, an SSH key pair, and `LATITUDESH_AUTH_TOKEN` exported.
+
+## Usage
+
+```
+export LATITUDESH_AUTH_TOKEN=...        # from the latitude.sh dashboard
+./deploy-latitude.sh up --name demo
+./deploy-latitude.sh status --name demo
+./deploy-latitude.sh ssh --name demo node1 'sudo systemctl status spinifex.target'
+./deploy-latitude.sh credentials --name demo
+./deploy-latitude.sh destroy --name demo
+```
+
+`./deploy-latitude.sh --help` lists the options. `--nodes` must be 1: three nodes need the VLAN and the Elastic IP move, which are not built.
+
+## What `up` does
+
+`terraform` → `host` (wait for SSH, cloud-init, `/dev/kvm`) → console check → `prep` (`resolv.conf`) → `disk` (`host-data-disk.sh`) → `install` → `ovn` → `firewall` → `start` → `verify` → `access` → AWS profile.
+
+- **Console check.** Before the first step that changes the node, `up` prints the recovery commands, runs `passwd` for the ubuntu user on the node, and waits for you to type `console-ok` after logging in over the serial console from another window. It asks again on every `up` that still has a changing step to run.
+- **Data disk.** `host-data-disk.sh` picks the blank NVMe by content and by-id EUI, never by `nvmeXn1` (L1, L11). It needs exactly one candidate of at least 100 GB with no signature, partitions, holders or mounts, or it stops and lists the disks (`--data-disk-id` chooses). It mounts by UUID with `nofail`.
+- **Firewall.** `setup.sh --firewall on` runs before Spinifex first starts, and the daemon writes `peers.nft` and loads the policy during the start. The start phase refuses to go on unless the active rules allow ssh: the accept rule in `spinifex.nft`, the defined ssh ports covering every port `sshd -T` reports, and `0.0.0.0/0` trusted in `custom.nft`.
+- **Proof.** `verify` and the access check run on every `up`, never skipped by a step marker. The access check opens a new ssh login, asks `https://<ip>:9999` for an answer, and confirms the firewall mode is on and the policy table is loaded.
+- **State** is in `~/.spinifex/latitude/<name>/`: Terraform state, `inputs`, `owned` (IDs of what exists), `step.*` markers, logs, the data disk UUID. `up`, `destroy` and `resolve-uncertain` hold a lock in the state directory (`lock.d`, a pid; a dead pid is taken over), so two runs cannot overlap. A failed `up` resumes at the failed step; delete a `step.<name>.nodeN` file to redo it. The AWS profile is `spinifex-latitude-<name>`, refused if either AWS file already has a section for it (any profile type) unless it equals exactly what `up` would write, or `--overwrite-profile` is given; the check runs again right before writing, with the CA at `~/.spinifex/latitude-<name>-ca.pem`. The secret key is written to `~/.aws/credentials` (0600, other profiles kept, the old file and `~/.aws/config` saved beside it) and never appears in a process argument.
+
+## Token handling
+
+The token is read from the environment by the provider and by the script's API calls. It is not a Terraform variable, is on no command line (curl reads it from stdin), is not sent to the node, and is written to no file. `TF_LOG` is unset by the script.
+
+## Recovery
+
+`up` prints the recovery text before the console check and again before the firewall step. With the firewall on, a lost ssh session is recovered from the latitude.sh serial console, `ssh <server_id>@<ip> -p 2222`, using the ubuntu password set in the console check. The printed disarm stops the daemon, deletes the `spinifex_filter` table and moves `peers.nft` aside. Do not reboot the node until `up` has finished or the disarm has been used: the boot unit reloads the policy from `peers.nft`.
+
+## destroy
+
+`destroy` builds the set to delete once, from the `owned` file plus the IDs in the Terraform state, shows it, and asks you to type the deployment name. It deletes exactly that set: `terraform destroy` first, then the API for anything left, in the order servers, Elastic IPs, keys, user data, project. If the set changes after the preview it stops and asks you to run it again.
+
+Objects whose names match the deployment but that were never recorded are listed as `untracked` and never deleted, and `destroy` exits 1 until you remove them by hand. If an `up` failed or was interrupted while creating (`uncertain` marker), an Elastic IP may exist that nothing can attribute: `destroy` lists everything it cannot attribute and gives no all-clear. The record is append-only; after you check the dashboard, `./deploy-latitude.sh resolve-uncertain --name <name>` clears it on a typed confirmation. The preview also lists every ID in the Terraform state that the API listing does not show (`absent`), and the state checksum is rechecked before `terraform destroy` runs. It also fails on any API error, unreadable list, unfollowed pagination or refused DELETE. Elastic IP release is asynchronous and only works once the address is active or in error, so a first sweep round may need to wait.
+
+## Unverified
+
+- Pagination: the API docs give only `page[size]` (default 20) and `page[number]` and no response metadata. The lists follow `links.next` if the body has one, and otherwise read page numbers until an empty page.
+
+- The OS slug `ubuntu_26_04_x64_lts` (the `latitudesh_operating_system` data source fails the plan if it is wrong; list slugs with `GET /plans/operating_systems`).
+- The DELETE paths for `/user_data/{id}` and `/projects/{id}`, and the shape of the `server` field on Elastic IP list items used by `leftovers`.
+- The `systemd-networkd-wait-online --any` drop-in path and effect (it first applies on the next boot).
+- That the Spinifex API certificate names the node's public address, which `write_profile` needs for `aws sts get-caller-identity` to pass.
+- Terraform owning the Elastic IP `server_id` fights a Spinifex-managed move; fine for one node, not for three.
+
+## Not exercised
+
+Any real API call, `up`, `destroy`, `host-data-disk.sh` on a host, the console check, the `--armed` start of `install-node.sh`, and the ssh-rule gate against a real `setup.sh` policy. Guest launch, public reachability of the Elastic IP, the Terraform workbooks and failover are not part of `up`; prove them by hand.

scripts/terraform/latitude-spx/cloud-init/user-data.yaml

@@ -0,0 +1,20 @@
+#cloud-config
+# Host preparation only. Spinifex is installed and formed over SSH by deploy-latitude.sh.
+# latitude.sh substitutes double-brace variables in user data, so this file holds none.
+write_files:
+  - path: /etc/cloud/cloud.cfg.d/99-spinifex-network.cfg
+    content: |
+      network: {config: disabled}
+  # The image's netplan lists eno2 with no address and without optional, so boot waits two minutes
+  # for it. --any returns once one link, eno1, is online; it changes the wait, not the network.
+  # UNVERIFIED: the binary path is the Ubuntu one and the drop-in first applies on the next boot.
+  - path: /etc/systemd/system/systemd-networkd-wait-online.service.d/10-spinifex-any.conf
+    content: |
+      [Service]
+      ExecStart=
+      ExecStart=/usr/lib/systemd/systemd-networkd-wait-online --any
+# runcmd carries on past a failing command, so each line stands alone.
+runcmd:
+  - [sh, -c, "systemctl disable --now unattended-upgrades.service apt-daily-upgrade.timer apt-daily.timer || true"]
+  - [sh, -c, "systemctl daemon-reload || true"]
+  - [sh, -c, "touch /var/lib/cloud/spinifex-host-ready"]

scripts/terraform/latitude-spx/compute.tf

@@ -0,0 +1,59 @@
+locals {
+  hostname_prefix = "spx-${var.deployment_name}-node"
+  project         = var.project != "" ? var.project : latitudesh_project.deploy[0].slug
+}
+
+# Both lookups fail the plan when a slug does not exist, which is the only check of the OS slug.
+data "latitudesh_operating_system" "os" {
+  slug = var.operating_system
+}
+
+data "latitudesh_plan" "plan" {
+  slug = var.plan
+}
+
+resource "latitudesh_project" "deploy" {
+  count             = var.project == "" ? 1 : 0
+  name              = "spinifex-${var.deployment_name}"
+  description       = "Spinifex deployment ${var.deployment_name}"
+  environment       = "Development"
+  provisioning_type = "on_demand"
+}
+
+# The key is team scoped. The server login user is ubuntu, not root.
+resource "latitudesh_ssh_key" "deploy" {
+  name       = "spinifex-${var.deployment_name}"
+  public_key = trimspace(file(pathexpand(var.ssh_public_key_path)))
+}
+
+# Host preparation only. Latitude substitutes double-brace variables inside user data, so the file
+# must hold none. Spinifex is installed over SSH by deploy-latitude.sh.
+resource "latitudesh_user_data" "host" {
+  description = "spinifex-${var.deployment_name}"
+  content     = filebase64("${path.module}/cloud-init/user-data.yaml")
+}
+
+# billing must be hourly: the default is monthly, which bills upfront with no refund.
+resource "latitudesh_server" "node" {
+  count            = var.node_count
+  project          = local.project
+  hostname         = "${local.hostname_prefix}${count.index + 1}"
+  site             = var.site
+  plan             = data.latitudesh_plan.plan.slug
+  operating_system = data.latitudesh_operating_system.os.slug
+  billing          = "hourly"
+  ssh_keys         = [latitudesh_ssh_key.deploy.id]
+  user_data        = latitudesh_user_data.host.id
+
+  timeouts = {
+    create = "60m"
+  }
+}
+
+# One Elastic IP routed to node 1. Terraform owning server_id fights a Spinifex-managed move of the
+# address (scout warning); that is fine for one node, and a multi-node deployment must stop managing
+# this attachment here before Spinifex moves it.
+resource "latitudesh_elastic_ip" "node1" {
+  project   = local.project
+  server_id = latitudesh_server.node[0].id
+}

scripts/terraform/latitude-spx/deploy-latitude.sh

@@ -0,0 +1,798 @@
+#!/usr/bin/env bash
+# deploy-latitude.sh — build a one-node Spinifex deployment on latitude.sh bare metal, or tear it down.
+#
+#   deploy-latitude.sh up                       one m4.metal.medium with an Elastic IP
+#   deploy-latitude.sh status                   nodes, addresses, what the deployment holds, steps done
+#   deploy-latitude.sh credentials              print the AWS access key and secret
+#   deploy-latitude.sh ssh [nodeN] [command]    SSH to a node, node1 by default
+#   deploy-latitude.sh destroy
+#   deploy-latitude.sh resolve-uncertain        clear the uncertain-create record after checking the dashboard
+#
+# Run it on your laptop with LATITUDESH_AUTH_TOKEN exported. OpenTofu, or Terraform when OpenTofu is
+# not installed, creates the project (unless --project), SSH key, user data, server and Elastic IP.
+# This script then works over SSH as the ubuntu user: it formats and mounts the blank data disk,
+# and drives install-node.sh (install, ovn, firewall arm, start, verify). Each step is recorded
+# under the state directory, so a rerun of up resumes at the step that failed.
+#
+# The token is read by the Terraform provider and by this script's own API calls from the
+# environment. It is never a Terraform variable, never on a command line (curl reads it from stdin),
+# and never sent to the node, so it is in no state file, log or user data.
+#
+# The AWS profile is "spinifex-latitude-<name>" (default name "default"). An existing profile of that
+# name that this script did not write is refused unless --overwrite-profile is given.
+
+set -euo pipefail
+shopt -s inherit_errexit
+# A provider trace log would carry the Authorization header.
+unset TF_LOG TF_LOG_PATH TF_LOG_CORE TF_LOG_PROVIDER
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SPINIFEX_SRC="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+LAT_API="https://api.latitude.sh"
+
+CMD="${1:-}"
+if [ $# -gt 0 ]; then shift; fi
+NODE_ARG=""
+NAME="default"
+SITE="DAL"
+PLAN="m4-metal-medium"
+OS="ubuntu_26_04_x64_lts"
+PROJECT=""
+DISTRO=""
+CHANNEL="latest"
+VERSION=""
+SPX_REGION="us-east-1"
+SPX_AZ="us-east-1a"
+SSH_KEY="$HOME/.ssh/id_ed25519"
+IMAGE="ubuntu-26.04-x86_64"
+DATA_DISK_ID=""
+STATE_ROOT="$HOME/.spinifex/latitude"
+NODES=1
+YES=0
+OVERWRITE_PROFILE=0
+
+usage() {
+    sed -n '2,23p' "$0" | sed 's/^# \{0,1\}//'
+    cat <<EOF
+
+Options:
+  --name NAME          deployment name (default $NAME), 1-16 characters; names every resource
+  --site SLUG          latitude.sh site (default $SITE); needs stock and Elastic IPs, not SAO
+  --plan SLUG          server plan (default $PLAN)
+  --os SLUG            operating system slug (default $OS, unverified)
+  --project ID|SLUG    deploy into this existing project; by default one is created and removed on destroy
+  --distro TARBALL     install this local release tarball
+  --channel CHANNEL    install from a release channel: latest (default) or dev
+  --version TAG        install this release instead of a channel
+  --spx-region NAME    Spinifex region (default $SPX_REGION)
+  --spx-az NAME        Spinifex availability zone (default $SPX_AZ)
+  --ssh-key PATH       private key; PATH.pub is installed (default $SSH_KEY)
+  --image NAME         AMI to import (default $IMAGE)
+  --no-image           import no AMI
+  --data-disk-id PATH  choose the data disk by its /dev/disk/by-id/nvme-eui.* path when several are blank
+  --nodes N            must be 1 for now
+  --overwrite-profile  replace an AWS profile of the same name that this script did not create
+  --yes                destroy without asking
+EOF
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --name) NAME="$2"; shift ;;
+        --site) SITE="$2"; shift ;;
+        --plan) PLAN="$2"; shift ;;
+        --os) OS="$2"; shift ;;
+        --project) PROJECT="$2"; shift ;;
+        --distro) DISTRO="$2"; shift ;;
+        --channel) CHANNEL="$2"; shift ;;
+        --version) VERSION="$2"; shift ;;
+        --spx-region) SPX_REGION="$2"; shift ;;
+        --spx-az) SPX_AZ="$2"; shift ;;
+        --ssh-key) SSH_KEY="$2"; shift ;;
+        --image) IMAGE="$2"; shift ;;
+        --no-image) IMAGE="" ;;
+        --data-disk-id) DATA_DISK_ID="$2"; shift ;;
+        --nodes) NODES="$2"; shift ;;
+        --overwrite-profile) OVERWRITE_PROFILE=1 ;;
+        --yes) YES=1 ;;
+        -h|--help) usage; exit 0 ;;
+        node[1-3])
+            case "$CMD" in ssh) ;; *) usage >&2; exit 2 ;; esac
+            [ -z "$NODE_ARG" ] || { usage >&2; exit 2; }
+            NODE_ARG="$1" ;;
+        *) if [ "$CMD" = ssh ]; then break; fi
+           echo "unknown option: $1" >&2; usage >&2; exit 2 ;;
+    esac
+    shift
+done
+
+log() { printf '[deploy %s] %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; }
+die() { log "ERROR: $*"; exit 1; }
+
+[[ "$NAME" =~ ^[a-z0-9][a-z0-9-]{0,15}$ ]] || die "--name must be 1-16 lowercase letters, digits or hyphens"
+[ "$NODES" = 1 ] || die "--nodes must be 1: three nodes need the VLAN and the Elastic IP move, which are not built yet"
+[[ "$SITE" =~ ^[A-Za-z0-9]+$ ]] || die "--site must be a site slug such as DAL"
+[[ "$SPX_REGION" =~ ^[a-z0-9-]+$ ]] || die "--spx-region must be lowercase letters, digits and hyphens"
+[[ "$SPX_AZ" =~ ^[a-z0-9-]+$ ]] || die "--spx-az must be lowercase letters, digits and hyphens"
+[[ "$CHANNEL" =~ ^[a-z0-9.-]+$ ]] || die "--channel must be lowercase letters, digits, dots and hyphens"
+[ -z "$VERSION" ] || [[ "$VERSION" =~ ^[A-Za-z0-9._-]+$ ]] || die "--version has characters a release tag does not"
+[ -z "$IMAGE" ] || [[ "$IMAGE" =~ ^[A-Za-z0-9._-]+$ ]] || die "--image has characters an image name does not"
+[ -z "$DATA_DISK_ID" ] || [[ "$DATA_DISK_ID" =~ ^/dev/disk/by-id/nvme-eui\.[A-Za-z0-9]+$ ]] || die "--data-disk-id must look like /dev/disk/by-id/nvme-eui.<hex>"
+case "$CMD" in up|destroy|status) [ -n "${LATITUDESH_AUTH_TOKEN:-}" ] || die "LATITUDESH_AUTH_TOKEN is not set" ;; esac
+if command -v tofu >/dev/null; then TF_BIN=tofu
+elif command -v terraform >/dev/null; then TF_BIN=terraform
+else die "neither OpenTofu (tofu) nor Terraform is installed"; fi
+
+STATE_DIR="$STATE_ROOT/$NAME"
+TF_STATE="$STATE_DIR/terraform.tfstate"
+KNOWN_HOSTS="$STATE_DIR/known_hosts"
+INPUTS="$STATE_DIR/inputs"
+OWNED="$STATE_DIR/owned"
+# PENDING exists while an apply runs. If the apply fails or is interrupted it is appended to UNCERTAIN, which
+# only resolve-uncertain removes: a create that reached latitude.sh but not the state cannot be attributed,
+# so destroy gives no all-clear while UNCERTAIN exists.
+UNCERTAIN="$STATE_DIR/uncertain"
+PENDING="$STATE_DIR/uncertain.pending"
+HOST_PREFIX="spx-$NAME-node"
+PROFILE="spinifex-latitude-$NAME"
+CA_FILE="$HOME/.spinifex/latitude-$NAME-ca.pem"
+SSH_OPTS=(-i "$SSH_KEY" -o BatchMode=yes -o IdentitiesOnly=yes -o ConnectTimeout=10
+    -o StrictHostKeyChecking=accept-new -o "UserKnownHostsFile=$KNOWN_HOSTS")
+mkdir -p "$STATE_DIR"
+chmod 0700 "$STATE_DIR"
+
+# After the first up, destroy, status and ssh use the project choice that up recorded.
+if [ -z "$PROJECT" ] && [ -f "$INPUTS" ]; then PROJECT="$(sed -n 's/^project=//p' "$INPUTS")"; fi
+
+# latitude METHOD PATH prints the response body and fails on a non-2xx. The token goes to curl on
+# stdin as a config line, so it is on no command line; printf is a shell builtin.
+latitude() {
+    local out code
+    out="$(printf 'header = "Authorization: Bearer %s"\n' "$LATITUDESH_AUTH_TOKEN" |
+        curl -sS -g -K - -w '\n%{http_code}' -X "$1" -H 'Accept: application/json' "$LAT_API$2")" || return 1
+    code="${out##*$'\n'}"
+    out="${out%$'\n'*}"
+    case "$code" in
+        2??) printf '%s' "$out" ;;
+        *) log "latitude.sh $1 $2 returned $code: $(printf '%s' "$out" | head -c 400)"; return 1 ;;
+    esac
+}
+
+# lat_list PATH [QUERY] prints a JSON array of every item of a paginated list. It follows links.next
+# until it is null; where the body gives no next key (the documented API has no pagination metadata, only page[size] and page[number]) it reads page numbers until an empty page, so a server-capped page cannot pass as the last. It fails on
+# an API error, on a body that is not {"data": [objects with an id and attributes]}, on a next link to
+# another host, on a page that repeats an earlier one, and past 200 pages.
+lat_list() {
+    local path="$1?${2:+$2&}page[size]=100&page[number]=1" out info n next page=1 all='[]'
+    while :; do
+        out="$(latitude GET "$path")" || return 1
+        info="$(python3 -I -c '
+import json,sys
+d = json.load(sys.stdin)
+data = d.get("data") if isinstance(d, dict) else None
+if not isinstance(data, list) or not all(isinstance(i, dict) and isinstance(i.get("id"), str) and isinstance(i.get("attributes"), dict) for i in data):
+    sys.exit("unexpected list schema")
+links = d.get("links")
+nxt = "?" if not isinstance(links, dict) or "next" not in links else (links["next"] or "-")
+print(len(data)); print(nxt)' <<<"$out")" || { log "latitude.sh list $1 returned an unexpected body"; return 1; }
+        n="$(sed -n 1p <<<"$info")" next="$(sed -n 2p <<<"$info")"
+        all="$(python3 -I -c '
+import json,sys
+a = json.loads(sys.argv[1]); new = json.load(sys.stdin)["data"]
+if new and any(i["id"] == new[0]["id"] for i in a): sys.exit("page repeats")
+print(json.dumps(a + new))' "$all" <<<"$out")" || { log "latitude.sh list $1 does not advance between pages"; return 1; }
+        page=$((page + 1))
+        [ "$page" -le 200 ] || { log "latitude.sh list $1 has more than 200 pages"; return 1; }
+        case "$next" in
+            -) break ;;
+            "?") if [ "$n" = 0 ]; then break; fi
+                 path="$1?${2:+$2&}page[size]=100&page[number]=$page" ;;
+            "$LAT_API"/*) path="${next#"$LAT_API"}" ;;
+            /*) path="$next" ;;
+            *) log "latitude.sh list $1 gave a next link to another host"; return 1 ;;
+        esac
+    done
+    printf '%s' "$all"
+}
+
+tf() {
+    "$TF_BIN" -chdir="$SCRIPT_DIR" "$@" -state="$TF_STATE" \
+        -var "deployment_name=$NAME" -var "site=$SITE" -var "plan=$PLAN" -var "operating_system=$OS" \
+        -var "project=$PROJECT" -var "ssh_public_key_path=$SSH_KEY.pub" -var "node_count=$NODES"
+}
+
+done_step() { [ -f "$STATE_DIR/step.$1" ]; }
+mark_step() { date -u +%FT%TZ > "$STATE_DIR/step.$1"; }
+
+# node_field FIELD prints FIELD of every node, in node order.
+node_field() {
+    "$TF_BIN" -chdir="$SCRIPT_DIR" output -state="$TF_STATE" -json nodes |
+        python3 -c 'import json,sys; [print(n[sys.argv[1]]) for n in json.load(sys.stdin)]' "$1"
+}
+
+# for_nodes_always FN runs FN on every node and records nothing.
+for_nodes_always() {
+    local fn="$1" i
+    for i in "${!IPS[@]}"; do
+        IP="${IPS[$i]}" SERVER_ID="${IDS[$i]}" NODE_NAME="node$((i + 1))"
+        "$fn"
+    done
+}
+
+# for_nodes STEP FN runs FN once per node with IP, SERVER_ID and NODE_NAME set, recording
+# STEP.<node> so a rerun skips the nodes already done.
+for_nodes() {
+    local step="$1" fn="$2" i
+    for i in "${!IPS[@]}"; do
+        IP="${IPS[$i]}" SERVER_ID="${IDS[$i]}" NODE_NAME="node$((i + 1))"
+        if done_step "$step.$NODE_NAME"; then continue; fi
+        "$fn"
+        mark_step "$step.$NODE_NAME"
+    done
+}
+
+# on_tty needs -tt before the destination so passwd gets a terminal.
+on_tty() { ssh -tt "${SSH_OPTS[@]}" "ubuntu@$IP" "$@"; }
+# shellcheck disable=SC2029  # the command is meant to expand here
+on() { ssh "${SSH_OPTS[@]}" "ubuntu@$IP" "$@"; }
+
+# acquire_lock stops two runs on one deployment, so resolve-uncertain cannot clear a record while an
+# apply is writing it. mkdir is atomic and works on macOS, which has no flock; a lock whose pid is gone is
+# stale and is taken over. Holding the lock means no apply is running, so a pending entry left by an
+# interrupted one is made permanent here.
+acquire_lock() {
+    local lock="$STATE_DIR/lock.d" pid
+    if ! mkdir "$lock" 2>/dev/null; then
+        pid="$(cat "$lock/pid" 2>/dev/null || true)"
+        if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then die "another run (pid $pid) is working on deployment '$NAME'"; fi
+        # A lock with no pid yet may be a run that has just made it: stale only once it is a minute old.
+        if [ -z "$pid" ] && [ -z "$(find "$lock" -maxdepth 0 -mmin +1 2>/dev/null)" ]; then die "another run is starting on deployment '$NAME'"; fi
+        rm -rf "$lock"
+        mkdir "$lock" || die "could not take the lock $lock"
+    fi
+    printf '%s\n' "$$" > "$lock/pid"
+    trap 'rm -rf "$STATE_DIR/lock.d"' EXIT
+    if [ -f "$PENDING" ]; then cat "$PENDING" >> "$UNCERTAIN"; rm -f "$PENDING"; fi
+}
+
+# ── up ───────────────────────────────────────────────────────────────────────
+
+record_inputs() {
+    local now
+    now="$(printf 'site=%s\nplan=%s\nos=%s\nproject=%s\nnodes=%s' "$SITE" "$PLAN" "$OS" "$PROJECT" "$NODES")"
+    if [ -f "$INPUTS" ]; then
+        [ "$(cat "$INPUTS")" = "$now" ] || die "inputs differ from the first up of '$NAME' (recorded in $INPUTS); use the same inputs, or destroy it first"
+    else
+        printf '%s\n' "$now" > "$INPUTS"
+    fi
+}
+
+preflight() {
+    [ -r "$SSH_KEY" ] || die "SSH key $SSH_KEY not readable"
+    [ -r "$SSH_KEY.pub" ] || die "SSH key $SSH_KEY.pub not readable"
+    command -v aws >/dev/null || die "the AWS CLI is not installed"
+    command -v python3 >/dev/null || die "python3 is not installed"
+    if [ -n "$DISTRO" ]; then [ -r "$DISTRO" ] || die "--distro $DISTRO not readable"; fi
+    latitude GET '/projects?page[size]=1' >/dev/null || die "the API refused the token (LATITUDESH_AUTH_TOKEN)"
+    if [ -n "$PROJECT" ]; then
+        lat_list /projects "filter[slug]=$PROJECT" | python3 -c '
+import json,sys
+sys.exit(0 if any(i["attributes"]["slug"] == sys.argv[1] for i in json.load(sys.stdin)) else 1)' "$PROJECT" ||
+            die "no project with slug $PROJECT; use its slug, or leave --project out to create one"
+    fi
+    record_inputs
+    # A fresh deployment refuses an existing profile now; a rerun is checked again, with the real values,
+    # right before the profile is written.
+    if [ ! -f "$TF_STATE" ] && [ "$OVERWRITE_PROFILE" != 1 ] && [ "$(profile_state)" != none ]; then
+        die "AWS profile $PROFILE already has a section in $(AWS_CRED_FILE) or $(AWS_CONF_FILE); pass --overwrite-profile to replace it (both files are saved first), or use another --name"
+    fi
+}
+
+provision() {
+    log "terraform apply ($PLAN at $SITE)"
+    "$TF_BIN" -chdir="$SCRIPT_DIR" init -input=false >/dev/null
+    # A pending entry left by an interrupted apply becomes a permanent one. Entries are only appended.
+    if [ -f "$PENDING" ]; then cat "$PENDING" >> "$UNCERTAIN"; rm -f "$PENDING"; fi
+    if [ -f "$UNCERTAIN" ]; then
+        log "an earlier create failed or was interrupted ($UNCERTAIN); destroy gives no all-clear until you check the dashboard and run: $0 resolve-uncertain --name $NAME"
+    fi
+    date -u +%FT%TZ > "$PENDING"
+    tf apply -input=false -auto-approve >"$STATE_DIR/apply.log" 2>&1 || {
+        tail -20 "$STATE_DIR/apply.log" >&2
+        { cat "$PENDING" 2>/dev/null || true; date -u +%FT%TZ; } >> "$UNCERTAIN"
+        rm -f "$PENDING"
+        record_owned || true
+        die "terraform apply failed; log in $STATE_DIR/apply.log. What was created is recorded in $OWNED and destroy removes it. A plan with no stock at the site fails here: try another --site"
+    }
+    record_owned
+    rm -f "$PENDING"
+}
+
+# record_owned adds the IDs of every latitude.sh object in the Terraform state to $OWNED, as
+# "kind id label". Terraform writes state as each object is created, so this also covers a partly
+# failed apply. destroy deletes only what is listed here.
+record_owned() {
+    [ -f "$TF_STATE" ] || return 0
+    local out
+    out="$("$TF_BIN" -chdir="$SCRIPT_DIR" show -json "$TF_STATE" | python3 -I -c '
+import json,sys
+kinds = {"latitudesh_server": "server", "latitudesh_elastic_ip": "elastic-ip", "latitudesh_ssh_key": "ssh-key",
+         "latitudesh_user_data": "user-data", "latitudesh_project": "project"}
+st = json.load(sys.stdin)
+root = (st.get("values") or {}).get("root_module") or {}
+for r in root.get("resources", []):
+    k = kinds.get(r.get("type"))
+    v = r.get("values") or {}
+    if k and v.get("id"):
+        print(k, v["id"], v.get("hostname") or v.get("address") or v.get("name") or v.get("description") or "-")')" ||
+        die "could not read the Terraform state to record what exists"
+    { [ ! -f "$OWNED" ] || cat "$OWNED"; printf '%s\n' "$out"; } | sed '/^$/d' | sort -u > "$OWNED.new"
+    chmod 0600 "$OWNED.new"
+    mv "$OWNED.new" "$OWNED"
+}
+
+# wait_host pins the host key only once the marker cloud-init writes exists.
+wait_host() {
+    log "waiting for SSH and cloud-init on $NODE_NAME"
+    local i
+    for i in $(seq 1 120); do
+        if ssh -i "$SSH_KEY" -o BatchMode=yes -o IdentitiesOnly=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=no \
+            -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR "ubuntu@$IP" \
+            'test -f /var/lib/cloud/spinifex-host-ready' 2>/dev/null; then break; fi
+        [ "$i" != 120 ] || die "host $IP was not ready after 20 minutes"
+        sleep 10
+    done
+    ssh-keygen -R "$IP" -f "$KNOWN_HOSTS" >/dev/null 2>&1 || true
+    on 'cloud-init status --wait >/dev/null; test -e /dev/kvm' || die "no /dev/kvm on $NODE_NAME"
+}
+
+# host_prep is the first change to the node, so it runs after the console gate. The image's resolv.conf
+# is not a link to systemd-resolved, so the node would bypass the split DNS Spinifex sets there for its
+# load balancer and EKS names.
+host_prep() {
+    on 'if [ ! -L /etc/resolv.conf ] && systemctl is-active -q systemd-resolved; then
+            sudo ln -sf ../run/systemd/resolve/stub-resolv.conf /etc/resolv.conf; fi' ||
+        die "could not point /etc/resolv.conf at systemd-resolved on $NODE_NAME"
+}
+
+# confirm_if_pending asks for the console only when a step that changes the node is still to run.
+confirm_if_pending() {
+    local st
+    for st in prep disk install ovn firewall start; do
+        if ! done_step "$st.$NODE_NAME"; then console_gate; return 0; fi
+    done
+}
+
+# console_gate runs before the first change to the node's disks, network or firewall. The person sets
+# the ubuntu password here (typed into passwd on the node, never seen by this script), then proves the
+# serial console works from another window, because that console is the only way back in.
+console_gate() {
+    log "console recovery check for $NODE_NAME, before any change to its disks or firewall"
+    print_recovery >&2
+    [ -r /dev/tty ] || die "no terminal: the console check needs a typed confirmation"
+    printf '\nSet the ubuntu password now (typed on the node, not seen by this script):\n' >&2
+    on_tty 'sudo passwd ubuntu' < /dev/tty || die "setting the ubuntu password failed"
+    printf '\nIn ANOTHER window run: ssh %s@%s -p 2222   and log in as ubuntu with that password.\n' "$SERVER_ID" "$IP" >&2
+    printf 'Type "console-ok" once you are logged in over the serial console: ' >&2
+    local answer
+    read -r answer < /dev/tty || die "no answer"
+    [ "$answer" = console-ok ] || die "not confirmed; nothing has been changed on $NODE_NAME. Rerun up once the console works"
+}
+
+# data_disk formats the blank data disk, chosen by content on the node, and records its UUID.
+data_disk() {
+    log "preparing the data disk on $NODE_NAME"
+    local tmp rc=0 uuid
+    scp -q "${SSH_OPTS[@]}" "$SCRIPT_DIR/host-data-disk.sh" "ubuntu@$IP:/tmp/host-data-disk.sh" || die "copy to $NODE_NAME failed"
+    tmp="$(mktemp "$STATE_DIR/data-disk.XXXXXX")"
+    on "sudo bash /tmp/host-data-disk.sh $DATA_DISK_ID" >"$tmp" 2>&1 || rc=$?
+    cat "$tmp" >&2
+    cat "$tmp" >> "$STATE_DIR/data-disk.$NODE_NAME.log"
+    uuid="$(sed -n 's/^DATA_UUID=//p' "$tmp" | tail -n1)"
+    rm -f "$tmp"
+    [ "$rc" = 0 ] || die "the data disk step failed (exit $rc); it may already have formatted the disk, so read the output above and $STATE_DIR/data-disk.$NODE_NAME.log before rerunning"
+    [[ "$uuid" =~ ^[0-9a-fA-F-]{16,36}$ ]] || die "the data disk step gave no UUID"
+    printf '%s\n' "$uuid" > "$STATE_DIR/data_uuid.$NODE_NAME"
+}
+
+# push_installer copies install-node.sh (and a local tarball with its setup.sh) to the node and
+# proves the installer arrived intact.
+push_installer() {
+    local want got
+    scp -q "${SSH_OPTS[@]}" "$SCRIPT_DIR/install-node.sh" "ubuntu@$IP:/tmp/install-node.sh" || die "copy to $NODE_NAME failed"
+    want="$(sha256sum "$SCRIPT_DIR/install-node.sh" | awk '{print $1}')"
+    got="$(on 'sha256sum /tmp/install-node.sh' | awk '{print $1}')"
+    [ "$want" = "$got" ] || die "install-node.sh changed in transit: local $want, node $got"
+    if [ -n "$DISTRO" ]; then
+        scp -q "${SSH_OPTS[@]}" "$DISTRO" "$SPINIFEX_SRC/scripts/setup.sh" "ubuntu@$IP:/tmp/" || die "copy of the tarball to $NODE_NAME failed"
+    fi
+}
+
+# install_args is every input install-node.sh records and a later phase must repeat.
+install_args() {
+    local uuid eip
+    uuid="$(cat "$STATE_DIR/data_uuid.$NODE_NAME")"
+    eip="$(cat "$STATE_DIR/eip")"
+    printf -- '--eip %s --data-uuid %s --node-name %s --spx-region %s --spx-az %s --channel %s' \
+        "$eip" "$uuid" "$NODE_NAME" "$SPX_REGION" "$SPX_AZ" "$CHANNEL"
+    if [ -n "$VERSION" ]; then printf -- ' --version %s' "$VERSION"; fi
+    if [ -n "$DISTRO" ]; then printf -- ' --distro /tmp/%s --setup-script /tmp/setup.sh' "$(basename "$DISTRO")"; fi
+}
+
+# run_phase PHASE [EXTRA...] runs one install-node.sh phase on the node as root.
+run_phase() {
+    local phase="$1"
+    shift
+    push_installer
+    log "install-node.sh --phase $phase on $NODE_NAME"
+    on "sudo bash /tmp/install-node.sh --phase $phase $(install_args) $*" 2>&1 | tee -a "$STATE_DIR/install-node.$NODE_NAME.log" >&2 ||
+        die "install-node.sh --phase $phase failed; the output above and $STATE_DIR/install-node.$NODE_NAME.log have it, and the node has /root/<step>.log"
+}
+
+phase_install() { run_phase install; }
+phase_ovn() { run_phase ovn; }
+
+# arm_firewall runs after the ovn phase and before Spinifex first starts, so the daemon never runs
+# unfiltered. The daemon writes peers.nft and loads the policy during its start. The start phase
+# refuses to go on unless the generated policy allows ssh: it reads the active rule, the defined ports
+# against sshd -T, and the 0.0.0.0/0 trust.
+arm_firewall() {
+    log "arming the firewall on $NODE_NAME before the first start"
+    print_recovery >&2
+    log "Do not reboot $NODE_NAME until this run has finished, or until the console disarm above has been used"
+    on 'sudo bash -c "env SETUP_STAGES=firewall /usr/local/share/spinifex/setup.sh --firewall on >/root/firewall.log 2>&1"' ||
+        die "arming the firewall failed; see /root/firewall.log on $NODE_NAME. Spinifex was not started"
+}
+
+phase_start() {
+    if [ -n "$IMAGE" ]; then run_phase start --armed --image "$IMAGE"; else run_phase start --armed; fi
+}
+
+phase_verify() { run_phase verify; }
+
+# check_access proves the firewalled node still lets a NEW ssh login and the API through from here.
+check_access() {
+    log "checking a new ssh login and the API through the firewall"
+    local i code=""
+    ssh "${SSH_OPTS[@]}" -o ControlMaster=no -o ControlPath=none "ubuntu@$IP" true ||
+        die "a new ssh login to $NODE_NAME failed after the firewall loaded; recover from the serial console (printed at the end of a good run; also: ssh $SERVER_ID@$IP -p 2222)"
+    for i in $(seq 1 12); do
+        code="$(curl -sk -m 10 -o /dev/null -w '%{http_code}' "https://$IP:9999/" || true)"
+        case "$code" in 2??|3??|4??) break ;; esac
+        sleep 5
+    done
+    case "$code" in
+        2??|3??|4??) ;;
+        *) die "the API on https://$IP:9999 did not answer from here (last status '$code'); ssh works, so check the firewall rules for tcp 9999" ;;
+    esac
+    on 'sudo bash -c "grep -qx on /etc/spinifex/firewall/mode && nft list table inet spinifex_filter >/dev/null"' ||
+        die "the firewall is not armed (mode not on or table inet spinifex_filter missing)"
+    log "access proven with the firewall armed"
+}
+
+AWS_CRED_FILE() { printf '%s' "${AWS_SHARED_CREDENTIALS_FILE:-$HOME/.aws/credentials}"; }
+AWS_CONF_FILE() { printf '%s' "${AWS_CONFIG_FILE:-$HOME/.aws/config}"; }
+
+# profile_state [KEY SECRET ENDPOINT] prints none when the profile has no section in either AWS file
+# (credentials: [name], config: [profile name]) whatever its type (static keys, SSO, credential_process).
+# With the values up would write it prints match when the sections equal exactly those, else conflict;
+# without them it prints exists. The secret reaches python on stdin.
+profile_state() {
+    printf '%s\n%s\n%s\n' "${1:-}" "${2:-}" "${3:-}" | python3 -I -c '
+import configparser, os, sys
+cred, conf, name, ca, region = sys.argv[1:6]
+key, secret, endpoint = sys.stdin.read().split("\n")[:3]
+def sect(path, sec):
+    cp = configparser.RawConfigParser()
+    cp.optionxform = str
+    if os.path.exists(path):
+        cp.read(path)
+    return dict(cp.items(sec)) if cp.has_section(sec) else None
+c = sect(cred, name)
+g = sect(conf, "profile " + name)
+if c is None and g is None:
+    print("none")
+elif not key:
+    print("exists")
+else:
+    want_c = {"aws_access_key_id": key, "aws_secret_access_key": secret}
+    want_g = {"region": region, "endpoint_url": endpoint, "ca_bundle": ca}
+    print("match" if c == want_c and g == want_g else "conflict")' "$(AWS_CRED_FILE)" "$(AWS_CONF_FILE)" "$PROFILE" "$CA_FILE" "$SPX_REGION"
+}
+
+# write_credentials KEY SECRET sets the profile's keys in the shared credentials file without putting
+# the secret in any process argument: it reaches python on stdin. Other profiles are kept (comments in
+# the file are not), the old file is saved beside it, and the new one is 0600.
+write_credentials() {
+    local code
+    code='
+import configparser, os, shutil, sys, time
+profile = sys.argv[1]
+key, secret = sys.stdin.read().split("\n")[:2]
+path = os.environ.get("AWS_SHARED_CREDENTIALS_FILE") or os.path.expanduser("~/.aws/credentials")
+os.umask(0o077)
+os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
+cp = configparser.RawConfigParser()
+cp.optionxform = str
+if os.path.exists(path):
+    cp.read(path)
+    shutil.copy2(path, "%s.pre-latitude.%d" % (path, int(time.time())))
+if not cp.has_section(profile):
+    cp.add_section(profile)
+cp.set(profile, "aws_access_key_id", key)
+cp.set(profile, "aws_secret_access_key", secret)
+cp.remove_option(profile, "aws_session_token")
+tmp = path + ".tmp"
+with open(tmp, "w") as f:
+    cp.write(f)
+os.chmod(tmp, 0o600)
+os.replace(tmp, path)
+'
+    printf '%s\n%s\n' "$1" "$2" | python3 -I -c "$code" "$PROFILE" || die "could not write the credentials file"
+}
+
+write_profile() {
+    log "writing the CA to $CA_FILE and AWS profile $PROFILE"
+    mkdir -p "$(dirname "$CA_FILE")"
+    local creds key secret ca_new
+    # The CA is fetched to a temporary file and checked first; the existing file is replaced only after
+    # the profile check passes, with the old one kept beside it.
+    ca_new="$(mktemp "$CA_FILE.new.XXXXXX")"
+    on 'sudo cat /etc/spinifex/ca.pem' > "$ca_new" || { rm -f "$ca_new"; die "could not read the node's CA"; }
+    grep -q -- '-----BEGIN CERTIFICATE-----' "$ca_new" || { rm -f "$ca_new"; die "the node's CA is empty or not a PEM certificate"; }
+    creds="$(on 'awk "/^\[spinifex\]/{f=1;next} /^\[/{f=0} f" ~/.aws/credentials')"
+    key="$(awk -F' *= *' '/aws_access_key_id/{print $2}' <<<"$creds")"
+    secret="$(awk -F' *= *' '/aws_secret_access_key/{print $2}' <<<"$creds")"
+    [ -n "$key" ] || die "no access key in the ubuntu user's ~/.aws/credentials on $NODE_NAME"
+    [ -n "$secret" ] || die "no [spinifex] credentials in the ubuntu user's ~/.aws/credentials on $NODE_NAME"
+    # Rechecked here, with the real values, immediately before anything is written.
+    case "$(profile_state "$key" "$secret" "https://$IP:9999")" in
+        none | match) ;;
+        *) [ "$OVERWRITE_PROFILE" = 1 ] || { rm -f "$ca_new"; die "AWS profile $PROFILE has a section in $(AWS_CRED_FILE) or $(AWS_CONF_FILE) that differs from what this run would write; pass --overwrite-profile to replace it (both files are saved first), or use another --name"; } ;;
+    esac
+    if [ -f "$CA_FILE" ] && ! cmp -s "$CA_FILE" "$ca_new"; then cp -p "$CA_FILE" "$CA_FILE.pre-latitude.$(date +%s)"; fi
+    chmod 0644 "$ca_new"
+    mv -f "$ca_new" "$CA_FILE"
+    write_credentials "$key" "$secret"
+    local cfg
+    cfg="$(AWS_CONF_FILE)"
+    if [ -f "$cfg" ]; then (umask 077; cp -p "$cfg" "$cfg.pre-latitude.$(date +%s)"); fi
+    aws configure set --profile "$PROFILE" region "$SPX_REGION"
+    aws configure set --profile "$PROFILE" endpoint_url "https://$IP:9999"
+    aws configure set --profile "$PROFILE" ca_bundle "$CA_FILE"
+    # The certificate must name the node's address for this to pass; that is not verified on latitude.sh before this script.
+    aws --profile "$PROFILE" sts get-caller-identity --query Account --output text >/dev/null ||
+        die "profile $PROFILE does not work against https://$IP:9999; check the API certificate names $IP"
+}
+
+print_recovery() {
+    printf '\nRecovery if ssh is ever lost (the firewall is on):\n'
+    printf '  Serial console: ssh %s@%s -p 2222   (needs an account SSH key and an OS user password: set one now with: %s ssh %s sudo passwd ubuntu)\n' \
+        "$SERVER_ID" "$IP" "$0" "$NODE_NAME"
+    printf '  Temporary disarm, in this order, from the console:\n'
+    printf '    sudo systemctl stop spinifex-daemon\n'
+    printf '    systemctl is-active spinifex-daemon      (expect: inactive)\n'
+    printf '    sudo nft delete table inet spinifex_filter\n'
+    printf '    sudo mv /etc/spinifex/firewall/peers.nft /root/peers.nft.disarmed\n'
+    printf '    sudo nft list tables      (must not list spinifex_filter)\n'
+    printf '  Do not reboot or restart Spinifex afterwards until a durable disarm is reviewed.\n'
+}
+
+cmd_up() {
+    preflight
+    if ! done_step terraform; then provision; mark_step terraform; fi
+    mapfile -t IPS < <(node_field public_ip)
+    mapfile -t IDS < <(node_field provider_host_id)
+    [ "${#IPS[@]}" = "$NODES" ] || die "the state holds ${#IPS[@]} node(s), not $NODES; destroy it or pass --nodes ${#IPS[@]}"
+    local eip
+    eip="$("$TF_BIN" -chdir="$SCRIPT_DIR" output -state="$TF_STATE" -raw elastic_ip)"
+    [[ "$eip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || die "Terraform reported no usable Elastic IP address ('$eip')"
+    printf '%s\n' "$eip" > "$STATE_DIR/eip"
+    for_nodes host wait_host
+    # The console is confirmed on every up that still has a risky step to run, not once per deployment.
+    for_nodes_always confirm_if_pending
+    for_nodes prep host_prep
+    for_nodes disk data_disk
+    for_nodes install phase_install
+    for_nodes ovn phase_ovn
+    for_nodes firewall arm_firewall
+    for_nodes start phase_start
+    # Never skipped: the claim of success is always checked live by this run.
+    for_nodes_always phase_verify
+    for_nodes_always check_access
+    IP="${IPS[0]}" SERVER_ID="${IDS[0]}" NODE_NAME=node1
+    write_profile
+    log "ready: https://$IP:9999 (profile $PROFILE)"
+    printf 'endpoint=https://%s:9999\nprofile=%s\nca=%s\nconsole=https://%s:3000/\nelastic_ip=%s\n' "$IP" "$PROFILE" "$CA_FILE" "$IP" "$eip"
+    local i
+    for i in "${!IPS[@]}"; do printf 'node%d=%s\n' "$((i + 1))" "${IPS[$i]}"; done
+    print_recovery
+    printf '\nVerified by this run: the install phases, the firewall policy loaded, the ssh login and API reachable from this host, and a working AWS profile. Not shown: a guest launch and public reachability of the Elastic IP; test those.\n'
+}
+
+# ── destroy ──────────────────────────────────────────────────────────────────
+
+# inventory prints "owned kind id label" for every object recorded in $OWNED that still exists, and
+# "untracked kind id label" for every object whose name matches this deployment but is not recorded.
+# With UNCERTAIN present it also prints "unattributed kind id label" for everything else in the account.
+# It reads full lists, fails on any API or schema error, and never reads a failure as empty.
+inventory() {
+    local d eip="" rc=0 uncertain=0
+    d="$(mktemp -d "$STATE_DIR/inv.XXXXXX")"
+    if [ -f "$STATE_DIR/eip" ]; then eip="$(cat "$STATE_DIR/eip")"; fi
+    if [ -f "$UNCERTAIN" ] || [ -f "$PENDING" ]; then uncertain=1; fi
+    {
+        lat_list /servers > "$d/server" &&
+            lat_list /elastic_ips > "$d/elastic-ip" &&
+            lat_list /ssh_keys > "$d/ssh-key" &&
+            lat_list /user_data > "$d/user-data" &&
+            lat_list /projects > "$d/project"
+    } || rc=1
+    if [ "$rc" = 0 ]; then
+        touch "$OWNED"
+        python3 -I -c '
+import json,sys,os
+d, owned_path, prefix, name, eip, own_project, uncertain = sys.argv[1:8]
+owned = set()
+for line in open(owned_path):
+    f = line.split()
+    if len(f) >= 2: owned.add((f[0], f[1]))
+seen = set()
+def label(a):
+    return a.get("hostname") or a.get("address") or a.get("name") or a.get("description") or "-"
+for kind in ("server", "elastic-ip", "ssh-key", "user-data", "project"):
+    for it in json.load(open(os.path.join(d, kind))):
+        a = it["attributes"]
+        if (kind, it["id"]) in owned:
+            seen.add((kind, it["id"]))
+            print("owned", kind, it["id"], label(a)); continue
+        match = False
+        if kind == "server": match = str(a.get("hostname", "")).startswith(prefix)
+        elif kind == "elastic-ip":
+            host = ((a.get("server") or {}).get("hostname")) or ""
+            match = bool(eip and a.get("address") == eip) or host.startswith(prefix)
+        elif kind == "ssh-key": match = a.get("name") == "spinifex-" + name
+        elif kind == "project": match = own_project == "1" and a.get("name") == "spinifex-" + name
+        elif kind == "user-data": match = a.get("description") == "spinifex-" + name
+        if match: print("untracked", kind, it["id"], label(a))
+        elif uncertain == "1": print("unattributed", kind, it["id"], label(a))
+for line in open(owned_path):
+    f = line.split()
+    if len(f) >= 2 and (f[0], f[1]) not in seen: print("absent", f[0], f[1], " ".join(f[2:]) or "-")
+' "$d" "$OWNED" "$HOST_PREFIX" "$NAME" "$eip" "$([ -z "$PROJECT" ] && echo 1 || echo 0)" "$uncertain" || rc=1
+    fi
+    find "$d" -type f -delete
+    rmdir "$d"
+    return "$rc"
+}
+
+# state_sum identifies the Terraform state that destroy will act on.
+state_sum() { if [ -f "$TF_STATE" ]; then sha256sum < "$TF_STATE"; else echo none; fi; }
+
+# owned_sum identifies the recorded set, so a change after the confirmation is noticed.
+owned_sum() { if [ -f "$OWNED" ]; then sha256sum < "$OWNED"; else echo none; fi; }
+
+cmd_destroy() {
+    local inv set before ssum kind id label i answer failed=0
+    # The deletion set is the recorded IDs plus the IDs in the Terraform state, built once and shown.
+    record_owned
+    ssum="$(state_sum)"
+    set="$(inventory)" || die "could not list the account; nothing was deleted and nothing is proven"
+    before="$(owned_sum)"
+    if [ "$YES" != 1 ]; then
+        log "deployment '$NAME': every recorded object will be deleted ('owned' exists, 'absent' is recorded but not in the listing, and terraform destroy acts on it too). 'untracked' and 'unattributed' ones will NOT be:"
+        printf '%s\n' "$set" >&2
+        read -r -p "Type the deployment name to confirm: " answer < /dev/tty
+        [ "$answer" = "$NAME" ] || die "not confirmed"
+    fi
+    record_owned
+    [ "$(state_sum)" = "$ssum" ] || die "the Terraform state changed after the preview; run destroy again to confirm what it holds now"
+    [ "$(owned_sum)" = "$before" ] || die "the set to delete changed after the preview; run destroy again to confirm the new set"
+    if [ -f "$TF_STATE" ]; then
+        log "terraform destroy"
+        "$TF_BIN" -chdir="$SCRIPT_DIR" init -input=false >/dev/null
+        tf destroy -input=false -auto-approve >"$STATE_DIR/destroy.log" 2>&1 || {
+            tail -20 "$STATE_DIR/destroy.log" >&2
+            die "terraform destroy failed; log in $STATE_DIR/destroy.log"
+        }
+    fi
+
+    # What Terraform did not remove: delete only members of the shown set, in dependency order. The paths
+    # are from the API reference (servers, elastic_ips, ssh_keys, user_data, projects), untested live.
+    for i in 1 2 3 4 5 6; do
+        inv="$(inventory)" || die "could not list what remains; treat everything as still billing and check the dashboard"
+        grep -q '^owned ' <<<"$inv" || break
+        for kind in server elastic-ip ssh-key user-data project; do
+            while read -r _ k id label; do
+                [ "$k" = "$kind" ] || continue
+                grep -q "^owned $kind $id " <<<"$set" || die "$kind $id is recorded but was not in the confirmed set; run destroy again"
+                log "deleting $kind $label ($id)"
+                case "$kind" in
+                    server) latitude DELETE "/servers/$id" >/dev/null || failed=1 ;;
+                    elastic-ip) latitude DELETE "/elastic_ips/$id" >/dev/null || failed=1 ;;
+                    ssh-key) latitude DELETE "/ssh_keys/$id" >/dev/null || failed=1 ;;
+                    user-data) latitude DELETE "/user_data/$id" >/dev/null || failed=1 ;;
+                    project) latitude DELETE "/projects/$id" >/dev/null || failed=1 ;;
+                esac
+            done < <(grep '^owned ' <<<"$inv")
+        done
+        sleep 15
+    done
+
+    inv="$(inventory)" || die "could not list what remains; treat everything as still billing and check the dashboard"
+    if grep -q '^owned ' <<<"$inv"; then
+        log "these recorded objects still exist and may be billing:"
+        grep '^owned ' <<<"$inv" >&2
+        exit 1
+    fi
+    [ "$failed" = 0 ] || die "a DELETE was refused (see above) although the objects are now absent; check the dashboard, then run destroy again"
+    if grep -q '^untracked ' <<<"$inv"; then
+        log "recorded objects are gone, but these name-matched objects are not recorded for this deployment. NOT deleted; check each in the dashboard and remove it by hand if it is yours:"
+        grep '^untracked ' <<<"$inv" >&2
+        exit 1
+    fi
+    if [ -f "$UNCERTAIN" ] || [ -f "$PENDING" ]; then
+        log "an earlier create failed or was interrupted before its IDs were recorded ($UNCERTAIN), so an Elastic IP or server may exist that this script cannot attribute. Everything it cannot attribute:"
+        grep '^unattributed ' <<<"$inv" >&2 || true
+        log "check the latitude.sh dashboard; delete anything of this deployment by hand, then run: $0 resolve-uncertain --name $NAME. No all-clear is given."
+        exit 1
+    fi
+    rm -f "$STATE_DIR"/step.* "$STATE_DIR"/eip "$STATE_DIR"/data_uuid.* "$INPUTS" "$OWNED" "$TF_STATE" "$TF_STATE.backup"
+    log "deployment '$NAME' destroyed: every recorded object is absent from complete account listings and none is name-matched. Objects never recorded and named differently are not covered by this check"
+}
+
+cmd_status() {
+    local nodes inv
+    if [ -f "$TF_STATE" ] && nodes="$("$TF_BIN" -chdir="$SCRIPT_DIR" output -state="$TF_STATE" -json nodes 2>/dev/null)"; then
+        python3 -I -c 'import json,sys; [print(n["name"], n["public_ip"]) for n in json.load(sys.stdin)]' <<<"$nodes"
+    fi
+    if inv="$(inventory)"; then
+        printf '%s\n' "$inv"
+    else
+        echo "UNKNOWN: the account could not be listed"
+    fi
+    find "$STATE_DIR" -maxdepth 1 -name 'step.*' | sort | sed 's/.*step\./done: /'
+}
+
+cmd_resolve_uncertain() {
+    local answer
+    [ -f "$UNCERTAIN" ] || [ -f "$PENDING" ] || { log "nothing uncertain is recorded for '$NAME'"; return 0; }
+    log "recorded uncertain creates:"
+    cat "$UNCERTAIN" "$PENDING" 2>/dev/null >&2 || true
+    [ -r /dev/tty ] || die "no terminal for the confirmation"
+    read -r -p 'Type "checked" once you have looked in the latitude.sh dashboard (servers, Elastic IPs, keys, user data, projects) and removed anything of this deployment that was not recorded: ' answer < /dev/tty
+    [ "$answer" = checked ] || die "not confirmed"
+    rm -f "$UNCERTAIN" "$PENDING"
+    log "uncertainty cleared for '$NAME'"
+}
+
+# node_ip NODE prints the public address of nodeN from the Terraform state.
+node_ip() {
+    [ -f "$TF_STATE" ] || die "no deployment '$NAME'; run up first"
+    local ip
+    ip="$(node_field public_ip | sed -n "${1#node}p")"
+    [ -n "$ip" ] || die "deployment '$NAME' has no $1"
+    printf '%s' "$ip"
+}
+
+cmd_credentials() {
+    IP="$(node_ip node1)"
+    on 'awk "/^\[spinifex\]/{f=1} /^\[/&&!/^\[spinifex\]/{f=0} f" ~/.aws/credentials'
+}
+
+cmd_ssh() {
+    IP="$(node_ip "${NODE_ARG:-node1}")"
+    # shellcheck disable=SC2029
+    ssh "${SSH_OPTS[@]}" "ubuntu@$IP" "$@"
+}
+
+case "$CMD" in up | destroy | resolve-uncertain) acquire_lock ;; esac
+case "$CMD" in
+    up) cmd_up ;;
+    destroy) cmd_destroy ;;
+    status) cmd_status ;;
+    credentials) cmd_credentials ;;
+    resolve-uncertain) cmd_resolve_uncertain ;;
+    ssh) cmd_ssh "$@" ;;
+    *) usage >&2; exit 2 ;;
+esac

scripts/terraform/latitude-spx/host-data-disk.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# host-data-disk.sh — format and mount the blank data disk at /var/lib/spinifex. Run as root ON the host.
+#
+#   host-data-disk.sh [/dev/disk/by-id/nvme-eui.<id>]
+#
+# The disk is chosen by content, never by its nvmeXn1 name, which swaps between boots. A candidate is a
+# whole-disk by-id EUI link whose device is not in the root filesystem's ancestry, has no partitions,
+# holders, swap or mounts, no filesystem signature and no partition table. Exactly one candidate must
+# remain, or the script stops and lists the disks; pass the by-id path to choose between several.
+# It never reformats: a rerun on a host already done prints the UUID and exits. The last line of stdout
+# is DATA_UUID=<uuid>, which the later install phases need.
+
+set -euo pipefail
+
+MP=/var/lib/spinifex
+LABEL=spinifex-data
+FSTAB_BAK=/etc/fstab.pre-spinifex
+MIN_BYTES="${MIN_DATA_BYTES:-100000000000}"
+WANT="${1:-}"
+
+die() { echo "STOP: $*" >&2; exit 1; }
+[ "$(id -u)" = 0 ] || die "run as root"
+
+# Resume: the mount is already there and is ours.
+if mp_uuid="$(findmnt --mountpoint "$MP" -no UUID 2>/dev/null)" && [ -n "$mp_uuid" ]; then
+    src="$(findmnt --mountpoint "$MP" -no SOURCE)"
+    [ "$(blkid -s LABEL -o value "$src")" = "$LABEL" ] || die "$MP is mounted from $src, which is not labelled $LABEL; not touching it"
+    grep -q "UUID=${mp_uuid}[[:space:]]" /etc/fstab || die "$MP is mounted but /etc/fstab has no UUID=$mp_uuid entry"
+    echo "already done: $MP is mounted from $src (UUID $mp_uuid)"
+    echo "DATA_UUID=$mp_uuid"
+    exit 0
+fi
+if grep -q "$MP" /etc/fstab; then die "/etc/fstab mentions $MP but nothing is mounted there; fix that by hand first"; fi
+[ ! -e "$MP" ] || [ -z "$(ls -A "$MP")" ] || die "$MP exists and is not empty"
+
+root_src="$(findmnt -no SOURCE /)" || die "cannot find the root source"
+root_anc="$(lsblk -s -nro NAME "$root_src")" || die "cannot read the root ancestry"
+[ -n "$root_anc" ] || die "empty root ancestry"
+
+# blank_reason ID prints why the disk is not a candidate and returns 1, or returns 0.
+blank_reason() {
+    local id="$1" dev name size kids hold sw mnt rc w
+    dev="$(readlink -e "$id")" || { echo "link does not resolve"; return 1; }
+    [ -b "$dev" ] || { echo "not a block device"; return 1; }
+    name="$(basename "$dev")"
+    if grep -qx "$name" <<<"$root_anc"; then echo "in the root ancestry"; return 1; fi
+    size="$(blockdev --getsize64 "$id")" || { echo "size unreadable"; return 1; }
+    [ "$size" -ge "$MIN_BYTES" ] || { echo "only $size bytes"; return 1; }
+    kids="$(lsblk -nro NAME "$id")" || { echo "children unreadable"; return 1; }
+    [ "$kids" = "$name" ] || { echo "has children"; return 1; }
+    hold="$(ls -A "/sys/block/$name/holders/")" || { echo "holders unreadable"; return 1; }
+    [ -z "$hold" ] || { echo "has holders"; return 1; }
+    sw="$(swapon --show=NAME --noheadings)" || { echo "swapon failed"; return 1; }
+    if grep -q "$name" <<<"$sw"; then echo "is swap"; return 1; fi
+    mnt="$(lsblk -nro MOUNTPOINTS "$id")" || { echo "mounts unreadable"; return 1; }
+    [ -z "${mnt//[[:space:]]/}" ] || { echo "is mounted"; return 1; }
+    set +e; blkid -p "$id" >/dev/null 2>&1; rc=$?; set -e
+    [ "$rc" -eq 2 ] || { echo "blkid -p says it has a signature (rc $rc)"; return 1; }
+    w="$(wipefs -n "$id")" || { echo "wipefs failed"; return 1; }
+    [ -z "$w" ] || { echo "wipefs finds signatures"; return 1; }
+    return 0
+}
+
+candidates=()
+shopt -s nullglob
+for id in /dev/disk/by-id/nvme-eui.*; do
+    case "$id" in *-part*) continue ;; esac
+    if [ -n "$WANT" ] && [ "$id" != "$WANT" ]; then continue; fi
+    if why="$(blank_reason "$id")"; then
+        candidates+=("$id")
+    else
+        echo "skip $id: $why"
+    fi
+done
+shopt -u nullglob
+
+if [ "${#candidates[@]}" -ne 1 ]; then
+    lsblk -o NAME,SIZE,TYPE,MODEL,SERIAL,FSTYPE,MOUNTPOINTS >&2 || true
+    die "expected exactly one blank data disk among the by-id EUI disks, found ${#candidates[@]}: ${candidates[*]:-none}"
+fi
+ID="${candidates[0]}"
+DEV="$(readlink -e "$ID")"
+echo "CHECKS PASSED: $ID -> $DEV"
+
+mkdir -p "$MP"
+mkfs.ext4 -q -L "$LABEL" "$ID"
+U="$(blkid -p -o value -s UUID "$ID")" || die "blkid UUID failed"
+[ -n "$U" ] || die "no UUID after mkfs"
+[ "$(blkid -p -o value -s TYPE "$ID")" = ext4 ] || die "not ext4 after mkfs"
+
+NEW="$(mktemp /etc/fstab.spinifex.XXXXXX)"
+cp /etc/fstab "$NEW"
+echo "UUID=$U $MP ext4 defaults,nofail,x-systemd.device-timeout=30s 0 2" >> "$NEW"
+findmnt --verify --tab-file "$NEW" || { rm -f "$NEW"; die "staged fstab failed verify, /etc/fstab untouched"; }
+BAK="$FSTAB_BAK.$(date +%s)"
+cp -a /etc/fstab "$BAK"
+trap 'cp -a "$BAK" /etc/fstab; systemctl daemon-reload; echo "ROLLED BACK /etc/fstab from $BAK" >&2' EXIT
+chmod 644 "$NEW"
+mv "$NEW" /etc/fstab
+systemctl daemon-reload
+mount "$MP"
+[ "$(findmnt --mountpoint "$MP" -no UUID)" = "$U" ] || die "wrong UUID at $MP"
+[ "$(findmnt --mountpoint "$MP" -no FSTYPE)" = ext4 ] || die "wrong fstype at $MP"
+[ "$(findmnt --mountpoint "$MP" -no MAJ:MIN | tr -d ' ')" = "$(lsblk -dno MAJ:MIN "$ID" | tr -d ' ')" ] || die "wrong device at $MP"
+trap - EXIT
+findmnt --mountpoint "$MP" -o TARGET,SOURCE,MAJ:MIN,UUID,FSTYPE,OPTIONS
+echo "STORAGE OK"
+echo "DATA_UUID=$U"

scripts/terraform/latitude-spx/outputs.tf

@@ -0,0 +1,25 @@
+output "deployment_name" {
+  value = var.deployment_name
+}
+
+output "site" {
+  value = var.site
+}
+
+output "project" {
+  value = local.project
+}
+
+# provider_host_id is the server ID. public_ip is the server's own /31 address, which SSH and the
+# Spinifex API use; the Elastic IP is separate.
+output "nodes" {
+  value = [for n in latitudesh_server.node : {
+    name             = trimprefix(n.hostname, "spx-${var.deployment_name}-")
+    provider_host_id = n.id
+    public_ip        = n.primary_ipv4
+  }]
+}
+
+output "elastic_ip" {
+  value = latitudesh_elastic_ip.node1.address
+}

scripts/terraform/latitude-spx/provider.tf

@@ -0,0 +1,4 @@
+# The token comes from LATITUDESH_AUTH_TOKEN in the environment, which the provider reads
+# itself (its auth_token argument documents that variable). It is never a variable here, so it
+# is not written to the state or to a tfvars file. The default project is set per resource.
+provider "latitudesh" {}

scripts/terraform/latitude-spx/terraform.tfvars.example

@@ -0,0 +1,5 @@
+deployment_name     = "l1"
+site                = "DAL"
+plan                = "m4-metal-medium"
+node_count          = 1
+ssh_public_key_path = "~/.ssh/id_ed25519.pub"

scripts/terraform/latitude-spx/variables.tf

@@ -0,0 +1,48 @@
+variable "deployment_name" {
+  type        = string
+  description = "Names every resource and the server hostnames, so two deployments in one account never collide."
+  validation {
+    # A hostname is at most 32 characters: "spx-" + name + "-node1".
+    condition     = can(regex("^[a-z0-9][a-z0-9-]{0,15}$", var.deployment_name))
+    error_message = "deployment_name must be 1-16 lowercase letters, digits or hyphens."
+  }
+}
+
+variable "project" {
+  type        = string
+  default     = ""
+  description = "ID or slug of an existing latitude.sh project to deploy into. Empty creates a project spinifex-<deployment_name> and removes it on destroy."
+}
+
+variable "site" {
+  type        = string
+  default     = "DAL"
+  description = "latitude.sh site slug. Pick one with stock for the plan and Elastic IP support; not SAO."
+}
+
+variable "plan" {
+  type        = string
+  default     = "m4-metal-medium"
+  description = "Server plan slug (the pricing page calls it m4.metal.medium)."
+}
+
+variable "operating_system" {
+  type        = string
+  default     = "ubuntu_26_04_x64_lts"
+  description = "OS slug. UNVERIFIED: the 26.04 slug follows the 24.04 pattern (ubuntu_24_04_x64_lts) and is checked by a data source at plan time."
+}
+
+variable "node_count" {
+  type    = number
+  default = 1
+  validation {
+    # Three nodes need the VLAN and the Elastic IP move, which are not built yet.
+    condition     = var.node_count == 1
+    error_message = "node_count must be 1."
+  }
+}
+
+variable "ssh_public_key_path" {
+  type    = string
+  default = "~/.ssh/id_ed25519.pub"
+}

scripts/terraform/latitude-spx/versions.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_version = ">= 1.5.0"
+  required_providers {
+    # Pinned exactly: 4.8.0 is the version whose schema these files were written against.
+    latitudesh = { source = "latitudesh/latitudesh", version = "= 4.8.0" }
+  }
+}

spinifex/agents/lambda/agent/bootstrap.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os/exec"
+	"path/filepath"
 	goruntime "runtime"
 	"syscall"
 	"time"
@@ -18,8 +19,8 @@ import (
 const sandboxID = 993
 
 // StartBootstrap starts the function's bootstrap at path as the sandbox uid and
-// gid with no supplementary groups, with no_new_privs, and with exactly env.
-// The caller waits on the returned command.
+// gid with no supplementary groups, with no_new_privs, and with exactly env,
+// in the bootstrap's own directory. The caller waits on the returned command.
 func StartBootstrap(ctx context.Context, path string, env []string, stdout, stderr io.Writer) (*exec.Cmd, error) {
 	cred := &syscall.Credential{Uid: sandboxID, Gid: sandboxID, Groups: []uint32{}}
 	return startBootstrap(ctx, path, env, stdout, stderr, cred)
@@ -30,6 +31,8 @@ func startBootstrap(ctx context.Context, path string, env []string, stdout, stde
 	// A nil Env inherits the agent's environment; the bootstrap gets only env.
 	cmd.Env = append([]string{}, env...)
 	cmd.Stdout, cmd.Stderr = stdout, stderr
+	// Packaged functions open their files relative to the task root.
+	cmd.Dir = filepath.Dir(path)
 	if cred != nil {
 		cmd.SysProcAttr = &syscall.SysProcAttr{Credential: cred}
 	}

spinifex/agents/lambda/agent/bootstrap_test.go

@@ -169,3 +169,11 @@ func TestStartBootstrapMissingExecutableFails(t *testing.T) {
 	_, err := startBootstrap(context.Background(), filepath.Join(t.TempDir(), "bootstrap"), nil, nil, nil, nil)
 	assert.Error(t, err)
 }
+
+func TestStartBootstrapRunsInItsOwnDirectory(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "data.txt"), []byte("packaged"), 0o644))
+	bootstrap := filepath.Join(dir, "bootstrap")
+	require.NoError(t, os.WriteFile(bootstrap, []byte("#!/bin/sh\ncat data.txt\n"), 0o755))
+	assert.Equal(t, "packaged", string(run(t, bootstrap, []string{"PATH=/bin:/usr/bin"}, false)))
+}

spinifex/bootstrap/config/config.go

@@ -5,6 +5,7 @@ package config
 import (
 	"fmt"
 	"log/slog"
+	"maps"
 	"net/netip"
 	"os"
 	"path/filepath"
@@ -355,6 +356,11 @@ type EBSConfig struct {
 	CacheSizeMB *int `json:"CacheSizeMB" mapstructure:"cache_size_mb"`
 }
 
+// SupportedEBSProvider reports whether name is a provider a node may run.
+func SupportedEBSProvider(name string) bool {
+	return name == EBSProviderViperblockd || name == EBSProviderQEMUNBD
+}
+
 // ResolvedProvider normalizes an empty Provider to EBSProviderViperblockd,
 // the default provider.
 func (c EBSConfig) ResolvedProvider() string {
@@ -801,6 +807,27 @@ func validateVultrPool(cc *ClusterConfig, p ExternalPool) error {
 	return nil
 }
 
+// validateSingleEBSProvider refuses nodes whose explicit [nodes.<name>.ebs]
+// providers differ. A peer stanza without one is unknown, not the default: the
+// generated config omits it, and the cluster's KV lock is the authoritative check.
+func validateSingleEBSProvider(cc *ClusterConfig) error {
+	var first, want string
+	for _, name := range slices.Sorted(maps.Keys(cc.Nodes)) {
+		if !viper.IsSet("nodes." + name + ".ebs.provider") {
+			continue
+		}
+		got := cc.Nodes[name].EBS.ResolvedProvider()
+		if first == "" {
+			first, want = name, got
+			continue
+		}
+		if got != want {
+			return fmt.Errorf("config: [nodes.%s.ebs] provider resolves to %q but [nodes.%s.ebs] resolves to %q; every node in a cluster must use the same EBS provider", first, want, name, got)
+		}
+	}
+	return nil
+}
+
 func validateClusterConfig(cc *ClusterConfig) error {
 	if viper.IsSet("network.external_dhcp") {
 		return fmt.Errorf("config: [network] external_dhcp is no longer supported; remove the key (static WAN-pool allocation only)")
@@ -809,9 +836,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		if viper.IsSet("nodes." + nodeName + ".vpcd.dhcp_bind_bridge") {
 			return fmt.Errorf("config: [nodes.%s.vpcd] dhcp_bind_bridge is no longer supported; remove the key (vpcd no longer runs a DHCP client)", nodeName)
 		}
-		switch nodeCfg.EBS.Provider {
-		case "", EBSProviderViperblockd, EBSProviderQEMUNBD:
-		case EBSProviderEmbedded:
+		switch {
+		case nodeCfg.EBS.Provider == "", SupportedEBSProvider(nodeCfg.EBS.Provider):
+		case nodeCfg.EBS.Provider == EBSProviderEmbedded:
 			// Refuse rather than silently upgrade: the embedded engine is gone,
 			// and starting anyway would migrate this node's volumes to
 			// ebsmetadata one-way without the operator ever asking for it.
@@ -829,6 +856,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 			return fmt.Errorf("config: [nodes.%s.ebs] cache_size_mb=%d must not be negative; use 0 to disable the cache, or remove the key for the default of %d", nodeName, *c, DefaultCacheSizeMB)
 		}
 	}
+	if err := validateSingleEBSProvider(cc); err != nil {
+		return err
+	}
 
 	if err := validateNorthstarCluster(cc); err != nil {
 		return err

spinifex/bootstrap/config/config_test.go

@@ -1108,6 +1108,45 @@ provider = "bogus"
 	assert.Contains(t, err.Error(), `provider="bogus"`)
 }
 
+func TestLoadConfig_EBSProvider_SingleProviderPerCluster(t *testing.T) {
+	stanza := func(node, provider string) string {
+		return fmt.Sprintf("[nodes.%s.ebs]\nprovider = %q\n", node, provider)
+	}
+	cases := []struct {
+		name    string
+		ebs     []string
+		wantErr string
+	}{
+		{"all unset", []string{"", "", ""}, ""},
+		{"explicit viperblockd beside unset peers", []string{"", stanza("n2", EBSProviderViperblockd), ""}, ""},
+		{"explicit qemunbd beside unset peers", []string{stanza("n1", EBSProviderQEMUNBD), "", ""}, ""},
+		{"all explicit qemunbd", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderQEMUNBD), stanza("n3", EBSProviderQEMUNBD)}, ""},
+		{"two explicit providers differ", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderViperblockd), ""},
+			`[nodes.n1.ebs] provider resolves to "qemunbd" but [nodes.n2.ebs] resolves to "viperblockd"`},
+		{"only the third node differs", []string{stanza("n1", EBSProviderViperblockd), stanza("n2", EBSProviderViperblockd), stanza("n3", EBSProviderQEMUNBD)},
+			`[nodes.n1.ebs] provider resolves to "viperblockd" but [nodes.n3.ebs] resolves to "qemunbd"`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			body := "node = \"n1\"\n"
+			for i, ebs := range tc.ebs {
+				body += fmt.Sprintf("\n[nodes.n%d]\nregion = \"us-east-1\"\n%s", i+1, ebs)
+			}
+			resetViper(t)
+			path := filepath.Join(t.TempDir(), "spinifex.toml")
+			require.NoError(t, os.WriteFile(path, []byte(body), 0600))
+
+			_, err := LoadConfig(path)
+			if tc.wantErr == "" {
+				require.NoError(t, err)
+				return
+			}
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.wantErr)
+		})
+	}
+}
+
 func TestEBSConfig_ResolvedTunablesDefault(t *testing.T) {
 	var c EBSConfig
 	assert.Equal(t, DefaultNBDKitThreads, c.ResolvedThreads())

spinifex/domains/ec2/accountcleanup/cleanup.go

@@ -69,6 +69,28 @@ var (
 	_ TagIndexOwner       = (*ec2tags.TagsServiceImpl)(nil)
 )
 
+// kindPrefixes maps each cleanable kind to its resource-ID prefix in the tag index.
+var kindPrefixes = map[string]string{
+	ec2v1.PlatformKindKeyPair:        keyPairIDPrefix,
+	ec2v1.PlatformKindPlacementGroup: placementGroupIDPrefix,
+	ec2v1.PlatformKindLaunchTemplate: launchTemplateIDPrefix,
+}
+
+// selectKinds returns the kinds a request cleans, in report order. No kinds
+// means all three; a kind that is not cleanable, such as the tag index, is refused.
+func selectKinds(requested []string) ([]string, error) {
+	all := []string{ec2v1.PlatformKindKeyPair, ec2v1.PlatformKindPlacementGroup, ec2v1.PlatformKindLaunchTemplate}
+	if len(requested) == 0 {
+		return all, nil
+	}
+	for _, kind := range requested {
+		if !slices.Contains(all, kind) {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The kind is not one of key-pair, placement-group or launch-template.")
+		}
+	}
+	return slices.DeleteFunc(all, func(kind string) bool { return !slices.Contains(requested, kind) }), nil
+}
+
 // DefaultSettleWindow is how old headerless residue must be before it is
 // removed; anything younger may be a create still in flight.
 const DefaultSettleWindow = 10 * time.Minute
@@ -102,6 +124,7 @@ const (
 // CleanupPlatform deletes the account's key pairs, empty placement groups and
 // launch templates, removes settled residue and orphaned tag-index entries,
 // then re-scans every owner store to build the result. It is safe to re-run.
+// Request.Kinds limits the run to those kinds and their tag-index entries.
 func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, error) {
 	if req == nil || req.AccountID == "" {
 		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "The request must specify an account ID.")
@@ -118,27 +141,54 @@ func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, e
 	acct := req.AccountID
 	res := &Result{AccountID: acct, CorrelationRef: req.CorrelationRef}
 
-	refs := c.readReferences(ctx, acct)
-	keyRun := c.cleanKeyPairs(ctx, acct, settledBefore, refs)
-	groupRun := c.cleanPlacementGroups(ctx, acct, refs)
-	templateRun := c.cleanLaunchTemplates(ctx, acct, settledBefore)
+	kinds, err := selectKinds(req.Kinds)
+	if err != nil {
+		return nil, err
+	}
+	var prefixes []string
+	for _, kind := range kinds {
+		prefixes = append(prefixes, kindPrefixes[kind])
+	}
+
+	var keyRun *keyPairRun
+	var groupRun *placementGroupRun
+	var templateRun *launchTemplateRun
+	if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) || slices.Contains(kinds, ec2v1.PlatformKindPlacementGroup) {
+		refs := c.readReferences(ctx, acct)
+		if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) {
+			keyRun = c.cleanKeyPairs(ctx, acct, settledBefore, refs)
+		}
+		if slices.Contains(kinds, ec2v1.PlatformKindPlacementGroup) {
+			groupRun = c.cleanPlacementGroups(ctx, acct, refs)
+		}
+	}
+	if slices.Contains(kinds, ec2v1.PlatformKindLaunchTemplate) {
+		templateRun = c.cleanLaunchTemplates(ctx, acct, settledBefore)
+	}
 
 	// Tags are listed before the owner re-scans, so a resource created in
 	// between is seen as owned and keeps its tags.
 	tagKind := c.startKind(ec2v1.PlatformKindTagIndex)
-	tagged, tagErr := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	tagged, tagErr := c.tags.ScanAccountResourceIDs(ctx, acct, prefixes...)
 	if tagErr != nil {
 		slog.WarnContext(ctx, "Platform cleanup: tag index listing failed", "err", tagErr)
 		tagKind.incomplete(ec2v1.PlatformReasonStoreListFailed)
 	}
 
-	owners := map[string]ownedIDs{
-		keyPairIDPrefix:        c.finishKeyPairs(ctx, acct, keyRun, res),
-		placementGroupIDPrefix: c.finishPlacementGroups(ctx, acct, groupRun, res),
-		launchTemplateIDPrefix: c.finishLaunchTemplates(ctx, acct, templateRun, res),
+	owners := map[string]ownedIDs{}
+	if keyRun != nil {
+		owners[keyPairIDPrefix] = c.finishKeyPairs(ctx, acct, keyRun, res)
+		res.Kinds = append(res.Kinds, keyRun.kind.report)
 	}
-	res.Kinds = []KindReport{keyRun.kind.report, groupRun.kind.report, templateRun.kind.report}
-	res.Kinds = append(res.Kinds, c.cleanTagIndex(ctx, acct, tagKind, tagged, tagErr == nil, owners, res))
+	if groupRun != nil {
+		owners[placementGroupIDPrefix] = c.finishPlacementGroups(ctx, acct, groupRun, res)
+		res.Kinds = append(res.Kinds, groupRun.kind.report)
+	}
+	if templateRun != nil {
+		owners[launchTemplateIDPrefix] = c.finishLaunchTemplates(ctx, acct, templateRun, res)
+		res.Kinds = append(res.Kinds, templateRun.kind.report)
+	}
+	res.Kinds = append(res.Kinds, c.cleanTagIndex(ctx, acct, tagKind, tagged, prefixes, tagErr == nil, owners, res))
 
 	slog.InfoContext(ctx, "Platform cleanup completed", "accountID", acct, "correlationRef", req.CorrelationRef,
 		"items", len(res.Items), "duration_ms", otelsetup.Millis(time.Since(start)))

spinifex/domains/ec2/accountcleanup/cleanup_test.go

@@ -775,3 +775,38 @@ func (f failingResidue) CleanupAccountResidue(ctx context.Context, acct string,
 	}
 	return &ec2launchtemplate.AccountCleanup{KeptTemplateIDs: scan.LiveTemplateIDs, OrphanBodiesFailed: scan.OrphanVersionBodies}, nil
 }
+
+// A run limited to one kind cleans that kind and its tag-index entries only:
+// the other kinds keep their resources and their tags, and are not reported.
+func TestCleanupPlatformLimitedToKindsLeavesTheRestAlone(t *testing.T) {
+	o := newOwners(t)
+	keyID := o.importKey(t, testAccount, "k1")
+	groupID := o.createGroup(t, testAccount, "g1")
+	o.tag(t, testAccount, keyID)
+
+	c := o.newCleanup()
+	c.now = func() time.Time { return time.Now().Add(time.Hour) }
+	res, err := c.CleanupPlatform(t.Context(), &Request{AccountID: testAccount, Kinds: []string{ec2v1.PlatformKindPlacementGroup}})
+	require.NoError(t, err)
+
+	assert.Equal(t, []Item{item(groupID, ec2v1.PlatformKindPlacementGroup, DispositionDeleted, "")}, res.Items)
+	assert.Equal(t, map[string]string{
+		ec2v1.PlatformKindPlacementGroup: ec2v1.PlatformStatusObservedClean,
+		ec2v1.PlatformKindTagIndex:       ec2v1.PlatformStatusObservedClean,
+	}, statuses(res))
+
+	keys, err := o.keys.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, keys.LiveKeyPairIDs, "an unselected kind is not touched")
+	tagged, err := o.tags.ScanAccountResourceIDs(t.Context(), testAccount, "key-")
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, tagged, "an unselected kind keeps its tags")
+}
+
+func TestCleanupPlatformRefusesAKindItCannotClean(t *testing.T) {
+	o := newOwners(t)
+	for _, kinds := range [][]string{{ec2v1.PlatformKindTagIndex}, {"volume"}} {
+		_, err := o.newCleanup().CleanupPlatform(t.Context(), &Request{AccountID: testAccount, Kinds: kinds})
+		require.Error(t, err, "%v", kinds)
+	}
+}

spinifex/domains/ec2/accountcleanup/kinds.go

@@ -282,15 +282,15 @@ func (c *Cleanup) finishLaunchTemplates(ctx context.Context, acct string, run *l
 // cleanTagIndex clears entries whose owner re-scan resolved and holds no
 // record for the ID, then re-lists the index to prove each clear. Entries of
 // resources that still exist are theirs, not residue, and are left alone.
-func (c *Cleanup) cleanTagIndex(ctx context.Context, acct string, kind *kindRun, tagged []string, listed bool, owners map[string]ownedIDs, res *Result) KindReport {
+func (c *Cleanup) cleanTagIndex(ctx context.Context, acct string, kind *kindRun, tagged, prefixes []string, listed bool, owners map[string]ownedIDs, res *Result) KindReport {
 	if listed {
-		c.clearOrphanTags(ctx, acct, kind, tagged, owners, res)
+		c.clearOrphanTags(ctx, acct, kind, tagged, prefixes, owners, res)
 	}
 	c.finishKind(kind)
 	return kind.report
 }
 
-func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRun, tagged []string, owners map[string]ownedIDs, res *Result) {
+func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRun, tagged, prefixes []string, owners map[string]ownedIDs, res *Result) {
 	// A resource this run reports as not deleted owns its tags even when its
 	// kind's re-scan failed.
 	reported := map[string]bool{}
@@ -317,7 +317,7 @@ func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRu
 			cleared = append(cleared, id)
 		}
 	}
-	after, err := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	after, err := c.tags.ScanAccountResourceIDs(ctx, acct, prefixes...)
 	if err != nil {
 		slog.WarnContext(ctx, "Platform cleanup: tag index re-scan failed", "err", err)
 		kind.incomplete(ec2v1.PlatformReasonStoreListFailed)

spinifex/domains/ec2/accountcleanup/types.go

@@ -7,6 +7,9 @@ import "time"
 type Request struct {
 	AccountID      string `json:"account_id"`
 	CorrelationRef string `json:"correlation_ref,omitempty"`
+
+	// Kinds limits the run to these platform kinds. Empty means all three.
+	Kinds []string `json:"kinds,omitempty"`
 }
 
 // Result is what the owner stores hold after one run. Kinds reuse the ec2.v1

spinifex/domains/ec2/instance/service_nats.go

@@ -1,80 +0,0 @@
-package instance
-
-import (
-	"context"
-	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/nats-io/nats.go"
-)
-
-// NATSInstanceService handles instance operations via NATS. Fan-out methods
-// (DescribeInstances, DescribeInstanceTypes) intentionally error — use the
-// gateway's scatter-gather helper for cluster-wide results.
-type NATSInstanceService struct {
-	natsConn *nats.Conn
-}
-
-var _ InstanceService = (*NATSInstanceService)(nil)
-
-// NewNATSInstanceService creates a new NATS-based instance service.
-func NewNATSInstanceService(conn *nats.Conn) InstanceService {
-	return &NATSInstanceService{natsConn: conn}
-}
-
-func (s *NATSInstanceService) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
-	if input == nil || input.InstanceType == nil {
-		return nil, fmt.Errorf("instance type is required")
-	}
-	topic := fmt.Sprintf("ec2.RunInstances.%s", aws.StringValue(input.InstanceType))
-	return natsmsg.NATSRequest[ec2.Reservation](ctx, s.natsConn, topic, input, 5*time.Minute, accountID)
-}
-
-// DescribeInstances is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstances(_ context.Context, _ *ec2.DescribeInstancesInput, _ string) (*ec2.DescribeInstancesOutput, error) {
-	return nil, fmt.Errorf("DescribeInstances is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-// DescribeInstanceTypes is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstanceTypes(_ context.Context, _ *ec2.DescribeInstanceTypesInput, _ string) (*DescribeInstanceTypesNodeOutput, error) {
-	return nil, fmt.Errorf("DescribeInstanceTypes is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-// DescribeInstanceStatus is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstanceStatus(_ context.Context, _ *ec2.DescribeInstanceStatusInput, _ string) (*ec2.DescribeInstanceStatusOutput, error) {
-	return nil, fmt.Errorf("DescribeInstanceStatus is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-func (s *NATSInstanceService) DescribeInstanceAttribute(ctx context.Context, input *ec2.DescribeInstanceAttributeInput, accountID string) (*ec2.DescribeInstanceAttributeOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstanceAttributeOutput](ctx, s.natsConn, "ec2.DescribeInstanceAttribute", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) DescribeStoppedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstancesOutput](ctx, s.natsConn, "ec2.DescribeStoppedInstances", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) DescribeTerminatedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstancesOutput](ctx, s.natsConn, "ec2.DescribeTerminatedInstances", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) ModifyInstanceAttribute(ctx context.Context, input *ec2.ModifyInstanceAttributeInput, accountID string) (*ec2.ModifyInstanceAttributeOutput, error) {
-	return natsmsg.NATSRequest[ec2.ModifyInstanceAttributeOutput](ctx, s.natsConn, "ec2.ModifyInstanceAttribute", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) ModifyInstanceMetadataOptions(ctx context.Context, input *ec2.ModifyInstanceMetadataOptionsInput, accountID string) (*ec2.ModifyInstanceMetadataOptionsOutput, error) {
-	return natsmsg.NATSRequest[ec2.ModifyInstanceMetadataOptionsOutput](ctx, s.natsConn, "ec2.ModifyInstanceMetadataOptions", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) StartStoppedInstance(ctx context.Context, input *StartStoppedInstanceInput, accountID string) (*StartStoppedInstanceOutput, error) {
-	return natsmsg.NATSRequest[StartStoppedInstanceOutput](ctx, s.natsConn, "ec2.StartStoppedInstance", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) TerminateStoppedInstance(ctx context.Context, input *TerminateStoppedInstanceInput, accountID string) (*TerminateStoppedInstanceOutput, error) {
-	return natsmsg.NATSRequest[TerminateStoppedInstanceOutput](ctx, s.natsConn, "ec2.TerminateStoppedInstance", input, 30*time.Second, accountID)
-}

spinifex/domains/ec2/instance/service_nats_test.go

@@ -1,364 +0,0 @@
-package instance
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/nats-io/nats-server/v2/server"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// startTestNATSServer starts an embedded NATS server for testing.
-func startTestNATSServer(t *testing.T) (*server.Server, string) {
-	t.Helper()
-	ns, nc := testutil.StartTestNATS(t)
-	_ = nc // connection managed by testutil cleanup
-	return ns, ns.ClientURL()
-}
-
-// createValidRunInstancesInput creates a valid RunInstancesInput for testing.
-func createValidRunInstancesInput() *ec2.RunInstancesInput {
-	return &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String("t3.micro"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-		KeyName:      aws.String("test-key-pair"),
-		SecurityGroupIds: []*string{
-			aws.String("sg-0123456789abcdef0"),
-		},
-		SubnetId: aws.String("subnet-6e7f829e"),
-		UserData: aws.String("#!/bin/bash\necho 'test'"),
-	}
-}
-
-// createValidReservation creates a valid ec2.Reservation response for testing.
-func createValidReservation() *ec2.Reservation {
-	return &ec2.Reservation{
-		ReservationId: aws.String("r-0123456789abcdef0"),
-		OwnerId:       aws.String("123456789012"),
-		Instances: []*ec2.Instance{
-			{
-				InstanceId:   aws.String("i-0123456789abcdef0"),
-				InstanceType: aws.String("t3.micro"),
-				ImageId:      aws.String("ami-0abcdef1234567890"),
-				State: &ec2.InstanceState{
-					Code: aws.Int64(0),
-					Name: aws.String("pending"),
-				},
-				PrivateIpAddress: aws.String("10.0.1.100"),
-				SubnetId:         aws.String("subnet-6e7f829e"),
-			},
-		},
-	}
-}
-
-// TestNATSInstanceService_RunInstances_Success tests successful RunInstances operation.
-func TestNATSInstanceService_RunInstances_Success(t *testing.T) {
-	// Skip if LOG_IGNORE is set
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	// Start test NATS server
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	// Create NATS connections
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err, "Failed to connect to NATS")
-	defer nc.Close()
-
-	// Create mock daemon subscriber that responds with valid reservation
-	// Subscribe to the per-instance-type topic (t3.micro from createValidRunInstancesInput)
-	mockReservation := createValidReservation()
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		// Validate that request is properly formatted
-		var input ec2.RunInstancesInput
-		err := json.Unmarshal(msg.Data, &input)
-		if err != nil {
-			t.Errorf("Mock daemon received invalid JSON: %v", err)
-			return
-		}
-
-		// Send back valid reservation
-		responseData, _ := json.Marshal(mockReservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err, "Failed to subscribe mock daemon")
-
-	// Create NATSInstanceService
-	service := NewNATSInstanceService(nc)
-	require.NotNil(t, service)
-
-	// Create valid input
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify success
-	require.NoError(t, err, "RunInstances should succeed")
-	require.NotNil(t, reservation, "Reservation should not be nil")
-
-	// Verify reservation contents
-	assert.Equal(t, *mockReservation.ReservationId, *reservation.ReservationId)
-	assert.Equal(t, *mockReservation.OwnerId, *reservation.OwnerId)
-	require.Len(t, reservation.Instances, 1)
-
-	instance := reservation.Instances[0]
-	assert.Equal(t, "i-0123456789abcdef0", *instance.InstanceId)
-	assert.Equal(t, "t3.micro", *instance.InstanceType)
-	assert.Equal(t, int64(0), *instance.State.Code)
-	assert.Equal(t, "pending", *instance.State.Name)
-}
-
-// TestNATSInstanceService_RunInstances_DaemonError tests error handling when daemon returns error.
-func TestNATSInstanceService_RunInstances_DaemonError(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that responds with error
-	// Subscribe to the per-instance-type topic for the requested type
-	_, err = nc.QueueSubscribe("ec2.RunInstances.invalid.type", "spinifex-workers", func(msg *nats.Msg) {
-		// Send back error response
-		errorResponse := awserrors.GenerateErrorPayload("InvalidInstanceType")
-		msg.Respond(errorResponse)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-
-	input := &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String("invalid.type"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-	}
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify error handling - error should be just the AWS error code for gateway lookup
-	require.Error(t, err, "Should return error when daemon returns error")
-	assert.Nil(t, reservation, "Reservation should be nil on error")
-	assert.Equal(t, "InvalidInstanceType", err.Error(), "Error should be the AWS error code")
-}
-
-// TestNATSInstanceService_RunInstances_NoSubscriber tests behavior when no daemon is subscribed.
-func TestNATSInstanceService_RunInstances_NoSubscriber(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// No subscriber - no daemon running
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	start := time.Now()
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-	duration := time.Since(start)
-
-	// Verify error behavior
-	// When no subscribers are available, NATS returns ErrNoResponders immediately
-	require.Error(t, err, "Should fail when no subscriber")
-	assert.Nil(t, reservation, "Reservation should be nil")
-	assert.ErrorIs(t, err, nats.ErrNoResponders, "Error should be ErrNoResponders")
-
-	// Should fail quickly (within 1 second), not wait for 30 second timeout
-	assert.Less(t, duration.Seconds(), 1.0, "Should fail immediately when no responders")
-}
-
-// TestNATSInstanceService_RunInstances_InvalidResponse tests handling of malformed responses.
-func TestNATSInstanceService_RunInstances_InvalidResponse(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that responds with invalid JSON
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		// Send back malformed JSON
-		msg.Respond([]byte(`{"invalid": json response`))
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify error handling
-	require.Error(t, err, "Should return error for invalid JSON")
-	assert.Nil(t, reservation, "Reservation should be nil")
-	assert.Contains(t, err.Error(), "failed to unmarshal", "Error should indicate unmarshal failure")
-}
-
-// TestNATSInstanceService_RunInstances_MarshalError tests handling when input marshaling fails.
-func TestNATSInstanceService_RunInstances_MarshalError(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	service := NewNATSInstanceService(nc)
-
-	// Nil input should be rejected before hitting NATS
-	reservation, err := service.RunInstances(context.Background(), nil, "123456789012")
-	require.Error(t, err, "Should handle nil input")
-	assert.Nil(t, reservation)
-	assert.Contains(t, err.Error(), "instance type is required")
-}
-
-// TestNATSInstanceService_RunInstances_MultipleInstances tests launching multiple instances.
-func TestNATSInstanceService_RunInstances_MultipleInstances(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that returns multiple instances
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		var input ec2.RunInstancesInput
-		json.Unmarshal(msg.Data, &input)
-
-		// Create reservation with count matching MinCount
-		count := int(*input.MinCount)
-		instances := make([]*ec2.Instance, count)
-		for i := range count {
-			instanceId := aws.String(aws.StringValue(aws.String(fmt.Sprintf("i-%d", i))))
-			instances[i] = &ec2.Instance{
-				InstanceId:   instanceId,
-				InstanceType: input.InstanceType,
-				ImageId:      input.ImageId,
-				State: &ec2.InstanceState{
-					Code: aws.Int64(0),
-					Name: aws.String("pending"),
-				},
-			}
-		}
-
-		reservation := &ec2.Reservation{
-			ReservationId: aws.String("r-0123456789abcdef0"),
-			Instances:     instances,
-		}
-
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-
-	// Request 3 instances
-	input := createValidRunInstancesInput()
-	input.MinCount = aws.Int64(3)
-	input.MaxCount = aws.Int64(3)
-
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify multiple instances returned
-	require.NoError(t, err)
-	require.NotNil(t, reservation)
-	assert.Len(t, reservation.Instances, 3, "Should have 3 instances")
-}
-
-// TestNATSInstanceService_RunInstances_QueueGroup tests load balancing with queue groups.
-func TestNATSInstanceService_RunInstances_QueueGroup(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create multiple mock daemon workers in same queue group
-	worker1Count := 0
-	worker2Count := 0
-
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		worker1Count++
-		reservation := createValidReservation()
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		worker2Count++
-		reservation := createValidReservation()
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Send multiple requests
-	requestCount := 10
-	for range requestCount {
-		reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-		require.NoError(t, err)
-		require.NotNil(t, reservation)
-	}
-
-	// Verify load was distributed (both workers should have handled some requests)
-	// With queue groups, NATS distributes messages across subscribers
-	t.Logf("Worker 1 handled: %d requests", worker1Count)
-	t.Logf("Worker 2 handled: %d requests", worker2Count)
-
-	assert.Positive(t, worker1Count, "Worker 1 should handle some requests")
-	assert.Positive(t, worker2Count, "Worker 2 should handle some requests")
-	assert.Equal(t, requestCount, worker1Count+worker2Count, "All requests should be handled")
-}

spinifex/domains/ecr/awsapi/gateway_image_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -25,7 +26,8 @@ import (
 // object store + memory meta) and its composed ECR JSON image-action service.
 func newImageGateway(t *testing.T) *ecrGW {
 	t.Helper()
-	meta := ecrdomain.NewMemoryMetaStore()
+	_, _, js := testutil.StartTestJetStream(t)
+	meta := ecrdomain.NewKVMetaStore(js)
 	reg := ecrregistry.NewRegistry(objectstore.NewMemoryObjectStore(), meta, ecrTestAccount)
 	gw := newECRGW(nil, awsapi.Deps{Registry: awsapi.NewRegistryActionService(reg, reg, reg, reg), LifecyclePreview: awsapi.NewLifecyclePreviewActionService(meta, reg)})
 	imageRegistries.Store(gw, reg)

spinifex/domains/ecr/meta.go

@@ -5,8 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"maps"
-	"slices"
 	"sort"
 	"strings"
 	"sync"
@@ -593,295 +591,6 @@ func (s *KVMetaStore) DeleteUpload(ctx context.Context, accountID, uploadID stri
 	return nil
 }
 
-// MemoryMetaStore is an in-memory MetaStore used by tests and single-process
-// dev runs. It is safe for concurrent use.
-type MemoryMetaStore struct {
-	mu        sync.Mutex
-	repos     map[string]map[string]RepoMeta     // account -> repo -> meta
-	policies  map[string]map[string][]byte       // account -> repo -> policyText
-	lifecycle map[string]map[string][]byte       // account -> repo -> lifecyclePolicyText
-	tags      map[string]map[string]string       // account -> repo|tag -> digest
-	manifests map[string]map[string]ManifestMeta // account -> repo|digest -> meta
-	uploads   map[string]map[string]uploadRev    // account -> id -> state+rev
-}
-
-type uploadRev struct {
-	state UploadState
-	rev   uint64
-}
-
-var _ MetaStore = (*MemoryMetaStore)(nil)
-
-// NewMemoryMetaStore constructs an empty in-memory MetaStore.
-func NewMemoryMetaStore() *MemoryMetaStore {
-	return &MemoryMetaStore{
-		repos:     make(map[string]map[string]RepoMeta),
-		policies:  make(map[string]map[string][]byte),
-		lifecycle: make(map[string]map[string][]byte),
-		tags:      make(map[string]map[string]string),
-		manifests: make(map[string]map[string]ManifestMeta),
-		uploads:   make(map[string]map[string]uploadRev),
-	}
-}
-
-func (m *MemoryMetaStore) PutRepo(_ context.Context, accountID string, meta RepoMeta) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.repos[accountID] == nil {
-		m.repos[accountID] = make(map[string]RepoMeta)
-	}
-	m.repos[accountID][meta.Name] = meta
-	return nil
-}
-
-func (m *MemoryMetaStore) GetRepo(_ context.Context, accountID, repo string) (RepoMeta, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	r, ok := m.repos[accountID][repo]
-	if !ok {
-		return RepoMeta{}, ErrNotFound
-	}
-	return r, nil
-}
-
-func (m *MemoryMetaStore) ListRepos(_ context.Context, accountID string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	return slices.Sorted(maps.Keys(m.repos[accountID])), nil
-}
-
-func (m *MemoryMetaStore) DeleteRepo(_ context.Context, accountID, repo string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if _, ok := m.repos[accountID][repo]; !ok {
-		return ErrNotFound
-	}
-	delete(m.repos[accountID], repo)
-	delete(m.policies[accountID], repo)
-	delete(m.lifecycle[accountID], repo)
-	for k := range m.tags[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo {
-			delete(m.tags[accountID], k)
-		}
-	}
-	for k := range m.manifests[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo {
-			delete(m.manifests[accountID], k)
-		}
-	}
-	return nil
-}
-
-func (m *MemoryMetaStore) ListManifests(_ context.Context, accountID, repo string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	var out []string
-	for k := range m.manifests[accountID] {
-		if r, d, ok := strings.Cut(k, "|"); ok && r == repo {
-			out = append(out, d)
-		}
-	}
-	sort.Strings(out)
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutRepoPolicy(_ context.Context, accountID, repo string, policyText []byte) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.policies[accountID] == nil {
-		m.policies[accountID] = make(map[string][]byte)
-	}
-	m.policies[accountID][repo] = policyText
-	return nil
-}
-
-func (m *MemoryMetaStore) GetRepoPolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.policies[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	return p, nil
-}
-
-func (m *MemoryMetaStore) DeleteRepoPolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.policies[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	delete(m.policies[accountID], repo)
-	return p, nil
-}
-
-func (m *MemoryMetaStore) PutLifecyclePolicy(_ context.Context, accountID, repo string, policyText []byte) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.lifecycle[accountID] == nil {
-		m.lifecycle[accountID] = make(map[string][]byte)
-	}
-	m.lifecycle[accountID][repo] = policyText
-	return nil
-}
-
-func (m *MemoryMetaStore) GetLifecyclePolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.lifecycle[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	return p, nil
-}
-
-func (m *MemoryMetaStore) DeleteLifecyclePolicy(_ context.Context, accountID, repo string) ([]byte, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	p, ok := m.lifecycle[accountID][repo]
-	if !ok {
-		return nil, ErrNotFound
-	}
-	delete(m.lifecycle[accountID], repo)
-	return p, nil
-}
-
-func (m *MemoryMetaStore) PutTag(_ context.Context, accountID, repo, tag, digest string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.tags[accountID] == nil {
-		m.tags[accountID] = make(map[string]string)
-	}
-	m.tags[accountID][repo+"|"+tag] = digest
-	return nil
-}
-
-func (m *MemoryMetaStore) GetTag(_ context.Context, accountID, repo, tag string) (string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	d, ok := m.tags[accountID][repo+"|"+tag]
-	if !ok {
-		return "", ErrNotFound
-	}
-	return d, nil
-}
-
-func (m *MemoryMetaStore) DeleteTag(_ context.Context, accountID, repo, tag string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	key := repo + "|" + tag
-	if _, ok := m.tags[accountID][key]; !ok {
-		return ErrNotFound
-	}
-	delete(m.tags[accountID], key)
-	return nil
-}
-
-func (m *MemoryMetaStore) ListTags(_ context.Context, accountID, repo string) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	var out []string
-	for k := range m.tags[accountID] {
-		if r, t, ok := strings.Cut(k, "|"); ok && r == repo {
-			out = append(out, t)
-		}
-	}
-	sort.Strings(out)
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutManifestMeta(_ context.Context, accountID, repo string, meta ManifestMeta) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.manifests[accountID] == nil {
-		m.manifests[accountID] = make(map[string]ManifestMeta)
-	}
-	m.manifests[accountID][repo+"|"+meta.Digest] = meta
-	return nil
-}
-
-func (m *MemoryMetaStore) GetManifestMeta(_ context.Context, accountID, repo, digest string) (ManifestMeta, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	meta, ok := m.manifests[accountID][repo+"|"+digest]
-	if !ok {
-		return ManifestMeta{}, ErrNotFound
-	}
-	return meta, nil
-}
-
-func (m *MemoryMetaStore) DeleteManifestMeta(_ context.Context, accountID, repo, digest string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	key := repo + "|" + digest
-	if _, ok := m.manifests[accountID][key]; !ok {
-		return ErrNotFound
-	}
-	delete(m.manifests[accountID], key)
-	return nil
-}
-
-func (m *MemoryMetaStore) ListReferrers(_ context.Context, accountID, repo, subjectDigest string) ([]ReferrerDescriptor, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	out := []ReferrerDescriptor{}
-	for k, meta := range m.manifests[accountID] {
-		if r, _, ok := strings.Cut(k, "|"); ok && r == repo && meta.SubjectDigest == subjectDigest {
-			out = append(out, meta.referrerDescriptor())
-		}
-	}
-	sort.Slice(out, func(i, j int) bool { return out[i].Digest < out[j].Digest })
-	return out, nil
-}
-
-func (m *MemoryMetaStore) PutUpload(_ context.Context, accountID, uploadID string, state UploadState) (uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if m.uploads[accountID] == nil {
-		m.uploads[accountID] = make(map[string]uploadRev)
-	}
-	rev := m.uploads[accountID][uploadID].rev + 1
-	m.uploads[accountID][uploadID] = uploadRev{state: state, rev: rev}
-	return rev, nil
-}
-
-func (m *MemoryMetaStore) GetUpload(_ context.Context, accountID, uploadID string) (UploadState, uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	u, ok := m.uploads[accountID][uploadID]
-	if !ok {
-		return UploadState{}, 0, ErrNotFound
-	}
-	return u.state, u.rev, nil
-}
-
-func (m *MemoryMetaStore) UpdateUpload(_ context.Context, accountID, uploadID string, state UploadState, rev uint64) (uint64, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	u, ok := m.uploads[accountID][uploadID]
-	if !ok {
-		return 0, ErrNotFound
-	}
-	if u.rev != rev {
-		return 0, ErrConflict
-	}
-	u.state = state
-	u.rev = rev + 1
-	m.uploads[accountID][uploadID] = u
-	return u.rev, nil
-}
-
-func (m *MemoryMetaStore) DeleteUpload(_ context.Context, accountID, uploadID string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if _, ok := m.uploads[accountID][uploadID]; !ok {
-		return ErrNotFound
-	}
-	delete(m.uploads[accountID], uploadID)
-	return nil
-}
-
 // mapKVErr normalizes JetStream KV errors to the MetaStore error vocabulary.
 func mapKVErr(err error) error {
 	if errors.Is(err, jetstream.ErrKeyNotFound) {

spinifex/domains/ecr/meta_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -61,8 +62,8 @@ func TestKVKeyHelpers(t *testing.T) {
 	assert.Equal(t, "sha256-abc", DigestToken("sha256:abc"))
 }
 
-func TestMemoryMetaStore_RepoRoundtrip(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_RepoRoundtrip(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	_, err := m.GetRepo(context.Background(), acct, "team/app")
@@ -80,8 +81,8 @@ func TestMemoryMetaStore_RepoRoundtrip(t *testing.T) {
 	assert.Equal(t, []string{"team/app", "team/web"}, repos)
 }
 
-func TestMemoryMetaStore_TagsAndManifests(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_TagsAndManifests(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	require.NoError(t, m.PutTag(context.Background(), acct, "r", "v1", "sha256:aaa"))
@@ -112,8 +113,8 @@ func TestMemoryMetaStore_TagsAndManifests(t *testing.T) {
 	assert.ErrorIs(t, m.DeleteManifestMeta(context.Background(), acct, "r", "sha256:ccc"), ErrNotFound)
 }
 
-func TestMemoryMetaStore_UploadCAS(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_UploadCAS(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	rev, err := m.PutUpload(context.Background(), acct, "u1", UploadState{RepoName: "r"})
@@ -143,8 +144,8 @@ func TestTrimSuffixMeta(t *testing.T) {
 	assert.Equal(t, "team/app", trimSuffixMeta("repos/team/app/meta"))
 }
 
-func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
-	m := NewMemoryMetaStore()
+func TestKVMetaStore_DeleteRepoAndListManifests(t *testing.T) {
+	m := newTestKVMetaStore(t)
 	const acct = "000000000000"
 
 	require.NoError(t, m.PutRepo(context.Background(), acct, RepoMeta{Name: "team/app", CreatedAt: time.Now()}))
@@ -156,7 +157,7 @@ func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
 
 	digs, err := m.ListManifests(context.Background(), acct, "team/app")
 	require.NoError(t, err)
-	assert.Equal(t, []string{"sha256:ccc"}, digs)
+	assert.Equal(t, []string{DigestToken("sha256:ccc")}, digs)
 
 	require.NoError(t, m.DeleteRepo(context.Background(), acct, "team/app"))
 	_, err = m.GetRepo(context.Background(), acct, "team/app")
@@ -180,50 +181,6 @@ func TestMemoryMetaStore_DeleteRepoAndListManifests(t *testing.T) {
 	assert.ErrorIs(t, m.DeleteRepo(context.Background(), acct, "team/ghost"), ErrNotFound)
 }
 
-func TestMemoryMetaStore_RepoPolicy(t *testing.T) {
-	m := NewMemoryMetaStore()
-	const acct = "000000000000"
-	const policy = `{"Version":"2012-10-17"}`
-
-	_, err := m.GetRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-	_, err = m.DeleteRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-
-	require.NoError(t, m.PutRepoPolicy(context.Background(), acct, "team/app", []byte(policy)))
-	got, err := m.GetRepoPolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(got))
-
-	deleted, err := m.DeleteRepoPolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(deleted))
-	_, err = m.GetRepoPolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-}
-
-func TestMemoryMetaStore_LifecyclePolicy(t *testing.T) {
-	m := NewMemoryMetaStore()
-	const acct = "000000000000"
-	const policy = `{"rules":[]}`
-
-	_, err := m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-	_, err = m.DeleteLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-
-	require.NoError(t, m.PutLifecyclePolicy(context.Background(), acct, "team/app", []byte(policy)))
-	got, err := m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(got))
-
-	deleted, err := m.DeleteLifecyclePolicy(context.Background(), acct, "team/app")
-	require.NoError(t, err)
-	assert.Equal(t, policy, string(deleted))
-	_, err = m.GetLifecyclePolicy(context.Background(), acct, "team/app")
-	assert.ErrorIs(t, err, ErrNotFound)
-}
-
 func TestManifestMeta_DecodesRecordWithoutReferrerFields(t *testing.T) {
 	var m ManifestMeta
 	require.NoError(t, json.Unmarshal([]byte(`{"digest":"sha256:ccc","mediaType":"application/vnd.oci.image.manifest.v1+json","size":10,"pushedAt":"2026-01-01T00:00:00Z","childDigests":["sha256:ddd"]}`), &m))
@@ -232,3 +189,9 @@ func TestManifestMeta_DecodesRecordWithoutReferrerFields(t *testing.T) {
 	assert.Empty(t, m.SubjectDigest)
 	assert.Empty(t, m.ArtifactType)
 }
+
+func newTestKVMetaStore(t *testing.T) *KVMetaStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	return NewKVMetaStore(js)
+}

spinifex/domains/ecr/referrers_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -30,11 +29,7 @@ func referrerMeta(digest, subject string) ManifestMeta {
 // its own referrers state.
 func referrerStores(t *testing.T) map[string]MetaStore {
 	t.Helper()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	return map[string]MetaStore{
-		"memory": NewMemoryMetaStore(),
-		"kv":     NewKVMetaStore(testutil.NewJetStream(t, nc)),
-	}
+	return map[string]MetaStore{"kv": newTestKVMetaStore(t)}
 }
 
 func TestListReferrers(t *testing.T) {

spinifex/domains/ecr/registry/lifecycle_sweep_test.go

@@ -50,7 +50,7 @@ func putPolicy(t *testing.T, reg *Registry, account, repo, policy string) {
 const untaggedExpirePolicy = `{"rules":[{"rulePriority":1,"selection":{"tagStatus":"untagged","countType":"sinceImagePushed","countUnit":"days","countNumber":7},"action":{"type":"expire"}}]}`
 
 func TestSweepRepo_SinceImagePushed(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -65,7 +65,7 @@ func TestSweepRepo_SinceImagePushed(t *testing.T) {
 }
 
 func TestSweepRepo_ImageCountMoreThan(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:a", now.AddDate(0, 0, -1), "a")
@@ -81,7 +81,7 @@ func TestSweepRepo_ImageCountMoreThan(t *testing.T) {
 }
 
 func TestSweepRepo_NoPolicy(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:x", now.AddDate(0, 0, -100))
@@ -92,7 +92,7 @@ func TestSweepRepo_NoPolicy(t *testing.T) {
 }
 
 func TestSweepRepo_Idempotent(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -105,7 +105,7 @@ func TestSweepRepo_Idempotent(t *testing.T) {
 }
 
 func TestSweepOnce_MultiAccount(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	const acct2 = "000000000001"
 	policy := `{"rules":[{"rulePriority":1,"selection":{"tagStatus":"any","countType":"imageCountMoreThan","countNumber":1},"action":{"type":"expire"}}]}`
@@ -125,7 +125,7 @@ func TestSweepOnce_MultiAccount(t *testing.T) {
 }
 
 func TestSweepOnce_AccountsError(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	s := NewLifecycleSweeper(reg, func() ([]string, error) {
 		return nil, assert.AnError
 	}, time.Hour)
@@ -133,7 +133,7 @@ func TestSweepOnce_AccountsError(t *testing.T) {
 }
 
 func TestNewLifecycleSweeper_DefaultInterval(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	s := NewLifecycleSweeper(reg, func() ([]string, error) { return nil, nil }, 0)
 	assert.Equal(t, DefaultLifecycleSweepInterval, s.interval)
 
@@ -147,7 +147,7 @@ func TestNewLifecycleSweeper_DefaultInterval(t *testing.T) {
 // TestSweeperRun drives the ticker loop: with a tiny interval and an expiring
 // image, the first tick must delete it; cancelling the context stops Run.
 func TestSweeperRun(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:olduntagged", now.AddDate(0, 0, -10))
@@ -184,7 +184,7 @@ func TestSweeperRun(t *testing.T) {
 // TestSweepRepo_InvalidStoredPolicy exercises the evaluate-error branch: a
 // malformed stored policy is logged and skipped, deleting nothing.
 func TestSweepRepo_InvalidStoredPolicy(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	repo := "team/app"
 	seedTimedImage(t, reg, testAccount, repo, "sha256:x", now.AddDate(0, 0, -100))
@@ -198,7 +198,7 @@ func TestSweepRepo_InvalidStoredPolicy(t *testing.T) {
 // TestSweepRepo_PolicyWithoutRepo exercises the ListImages-not-found branch: a
 // policy stored for a repo with no manifests yields nothing to expire.
 func TestSweepRepo_PolicyWithoutRepo(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	now := time.Now().UTC()
 	putPolicy(t, reg, testAccount, "ghost", untaggedExpirePolicy)
 

spinifex/domains/ecr/registry/registry_ensurebucket_test.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"testing"
 
-	"github.com/mulgadc/spinifex/spinifex/domains/ecr"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -32,7 +31,7 @@ func (c *countingStore) EnsureBucket(ctx context.Context, bucket string) error {
 
 func TestRegistry_EnsureBucketFailureReturns500(t *testing.T) {
 	store := &countingStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), ensureErr: errors.New("predastore down")}
-	reg := NewRegistry(store, ecr.NewMemoryMetaStore(), testAccount)
+	reg := NewRegistry(store, newTestMetaStore(t), testAccount)
 
 	w := do(reg, http.MethodGet, "/v2/myrepo/tags/list", nil, nil)
 	assert.Equal(t, http.StatusInternalServerError, w.Code)
@@ -40,7 +39,7 @@ func TestRegistry_EnsureBucketFailureReturns500(t *testing.T) {
 
 func TestRegistry_EnsureBucketCachedAfterSuccess(t *testing.T) {
 	store := &countingStore{MemoryObjectStore: objectstore.NewMemoryObjectStore()}
-	reg := NewRegistry(store, ecr.NewMemoryMetaStore(), testAccount)
+	reg := NewRegistry(store, newTestMetaStore(t), testAccount)
 
 	do(reg, http.MethodGet, "/v2/myrepo/tags/list", nil, nil)
 	do(reg, http.MethodGet, "/v2/other/tags/list", nil, nil)

spinifex/domains/ecr/registry/registry_image_test.go

@@ -44,7 +44,7 @@ func storeLayersManifest(t *testing.T, reg *Registry, repo, tag string, layers .
 }
 
 func TestDeleteImage_ReclaimsBlobsKeepsShared(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	shared := pushBlob(t, reg, repo, []byte("shared-layer-bytes"))
 	exclusive := pushBlob(t, reg, repo, []byte("exclusive-layer-bytes"))
@@ -85,7 +85,7 @@ func seedImage(t *testing.T, reg *Registry, repo, tag string) ([]byte, string) {
 }
 
 func TestStoreAndGetManifest_RoundTrip(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	manifest, digest := seedImage(t, reg, "team/app", "v1")
 
 	// Fetch by tag.
@@ -114,7 +114,7 @@ func TestStoreAndGetManifest_RoundTrip(t *testing.T) {
 }
 
 func TestListImages_TaggedAndUntagged(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	seedImage(t, reg, repo, "v1")
 	seedImage(t, reg, repo, "v2")
@@ -176,7 +176,7 @@ func TestListImages_KVDigestForm(t *testing.T) {
 }
 
 func TestDeleteImage_ByTagAndByDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	_, digest := seedImage(t, reg, repo, "v1")
 	seedImage(t, reg, repo, "v2") // shares nothing; distinct digest
@@ -205,7 +205,7 @@ func TestDeleteImage_ByTagAndByDigest(t *testing.T) {
 }
 
 func TestDeleteImage_ByDigestClearsTags(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	_, digest := seedImage(t, reg, repo, "v1")
 	// Add a second tag pointing at the same digest.
@@ -221,7 +221,7 @@ func TestDeleteImage_ByDigestClearsTags(t *testing.T) {
 }
 
 func TestStoreManifest_ImmutableTag(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	manA := fmt.Appendf(nil, `{"schemaVersion":2,"mediaType":"%s","layers":[{"digest":"%s"}]}`,
 		mediaTypeDockerManifest, pushBlob(t, reg, repo, []byte("layer-a")))
@@ -254,7 +254,7 @@ func TestStoreManifest_ImmutableTag(t *testing.T) {
 }
 
 func TestStoreManifest_BadDigestReference(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfgDg := pushBlob(t, reg, repo, []byte("cfg"))
 	manifest := fmt.Appendf(nil,

spinifex/domains/ecr/registry/registry_referrer_test.go

@@ -38,7 +38,7 @@ func storedMeta(t *testing.T, reg *Registry, body string) (subject, artifactType
 }
 
 func TestPutManifest_SubjectRecordedAndAdvertised(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 
 	tests := []struct {
@@ -65,7 +65,7 @@ func TestPutManifest_SubjectRecordedAndAdvertised(t *testing.T) {
 }
 
 func TestPutManifest_IndexSubject(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"manifests":[],"subject":{"mediaType":%q,"digest":%q,"size":10}}`,
@@ -81,7 +81,7 @@ func TestPutManifest_IndexSubject(t *testing.T) {
 }
 
 func TestPutManifest_NoSubjectLeavesFieldsEmpty(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"artifactType":%q,"config":{"mediaType":%q,"digest":%q,"size":3},"layers":[]}`,
 		mediaTypeOCIManifest, sigArtifactType, sigConfigType, cfg)
@@ -97,7 +97,7 @@ func TestPutManifest_NoSubjectLeavesFieldsEmpty(t *testing.T) {
 }
 
 func TestPutManifest_MalformedSubjectDigestRejected(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := imageWithSubject(cfg, "sha256:nothex", "")
 
@@ -111,7 +111,7 @@ func TestPutManifest_MalformedSubjectDigestRejected(t *testing.T) {
 }
 
 func TestPutManifest_IndexWithContentTypeParameterGetsNoConfigArtifactType(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	body := fmt.Sprintf(`{"schemaVersion":2,"mediaType":%q,"manifests":[],"subject":{"mediaType":%q,"digest":%q,"size":10}}`,
@@ -128,7 +128,7 @@ func TestPutManifest_IndexWithContentTypeParameterGetsNoConfigArtifactType(t *te
 }
 
 func TestDeleteReferrerManifest_RemovesItFromTheSubjectListing(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	cfg := pushBlob(t, reg, referrerRepo, []byte("cfg"))
 	body := imageWithSubject(cfg, absentSubjectDgt, sigArtifactType)
 	dg := digestOf([]byte(body))
@@ -183,7 +183,7 @@ func getReferrersBody(t *testing.T, reg *Registry, path string) (*httptest.Respo
 }
 
 func TestGetReferrers_ListsReferrersAsAnOCIIndex(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dgA := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	dgB := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, "application/vnd.example.sbom.v1", "cfgB")
 
@@ -202,7 +202,7 @@ func TestGetReferrers_ListsReferrersAsAnOCIIndex(t *testing.T) {
 }
 
 func TestGetReferrers_NoReferrersIsAnEmptyIndexNotAnError(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	w := do(reg, http.MethodGet, "/v2/"+referrerRepo+"/referrers/"+absentSubjectDgt, nil, nil)
@@ -212,7 +212,7 @@ func TestGetReferrers_NoReferrersIsAnEmptyIndexNotAnError(t *testing.T) {
 }
 
 func TestGetReferrers_ArtifactTypeFilter(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dgSig := pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	pushReferrer(t, reg, referrerRepo, absentSubjectDgt, "application/vnd.example.sbom.v1", "cfgB")
 
@@ -230,7 +230,7 @@ func TestGetReferrers_ArtifactTypeFilter(t *testing.T) {
 }
 
 func TestGetReferrers_MalformedDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 
 	w := do(reg, http.MethodGet, "/v2/"+referrerRepo+"/referrers/not-a-digest", nil, nil)
@@ -239,14 +239,14 @@ func TestGetReferrers_MalformedDigest(t *testing.T) {
 }
 
 func TestGetReferrers_UnknownRepository(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/ghost/referrers/"+absentSubjectDgt, nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "NAME_UNKNOWN")
 }
 
 func TestGetReferrers_DoesNotCrossRepositories(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	pushReferrer(t, reg, referrerRepo, absentSubjectDgt, sigArtifactType, "cfgA")
 	seedRepo(t, reg, "team/other")
 
@@ -256,7 +256,7 @@ func TestGetReferrers_DoesNotCrossRepositories(t *testing.T) {
 }
 
 func TestReferrersRoute_OnlyGET(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 	for _, method := range []string{http.MethodPut, http.MethodDelete, http.MethodPost, http.MethodPatch} {
 		w := do(reg, method, "/v2/"+referrerRepo+"/referrers/"+absentSubjectDgt, []byte("{}"), nil)
@@ -272,7 +272,7 @@ func (failingReferrerStore) ListReferrers(context.Context, string, string, strin
 }
 
 func TestGetReferrers_MetadataFailureIsA5xxNotA404(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, referrerRepo)
 	reg.Meta = failingReferrerStore{reg.Meta}
 
@@ -310,7 +310,7 @@ func referrerDigests(t *testing.T, reg *Registry, repo, subject string) []string
 }
 
 func TestDeleteManifest_SubjectDeleteRemovesItsReferrers(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	sbom := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "sbom")
@@ -326,7 +326,7 @@ func TestDeleteManifest_SubjectDeleteRemovesItsReferrers(t *testing.T) {
 }
 
 func TestDeleteManifest_NestedReferrersAreRemoved(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	sigOfSig := pushReferrer(t, reg, referrerRepo, sig, sigArtifactType, "sig-of-sig")
@@ -341,7 +341,7 @@ func TestDeleteManifest_NestedReferrersAreRemoved(t *testing.T) {
 }
 
 func TestDeleteManifest_ReferrerDeleteLeavesTheSubject(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	other := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "sbom")
@@ -355,7 +355,7 @@ func TestDeleteManifest_ReferrerDeleteLeavesTheSubject(t *testing.T) {
 }
 
 func TestDeleteManifest_TagDeleteDoesNotCascade(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	sig := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	require.NoError(t, reg.Meta.PutTag(context.Background(), testAccount, referrerRepo, "v1", subject))
@@ -368,7 +368,7 @@ func TestDeleteManifest_TagDeleteDoesNotCascade(t *testing.T) {
 }
 
 func TestDeleteManifest_CascadeStaysInsideTheRepository(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "sig")
 	elsewhere := pushReferrer(t, reg, "team/other", subject, sigArtifactType, "sig-other")
@@ -394,7 +394,7 @@ func (s failingDeleteStore) DeleteManifestMeta(ctx context.Context, account, rep
 }
 
 func TestDeleteManifest_ReferrerFailureDoesNotFailTheSubjectDelete(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	subject := pushSubjectImage(t, reg, referrerRepo, "subject")
 	stuck := pushReferrer(t, reg, referrerRepo, subject, sigArtifactType, "stuck")
 	fine := pushReferrer(t, reg, referrerRepo, subject, "application/vnd.example.sbom.v1", "fine")

spinifex/domains/ecr/registry/registry_test.go

@@ -2,6 +2,7 @@ package ecrregistry
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/internal/testkit"
 
 	"crypto/sha256"
 	"encoding/hex"
@@ -21,8 +22,15 @@ import (
 
 const testAccount = "000000000000"
 
-func newTestRegistry() *Registry {
-	return NewRegistry(objectstore.NewMemoryObjectStore(), ecr.NewMemoryMetaStore(), testAccount)
+func newTestRegistry(t *testing.T) *Registry {
+	t.Helper()
+	return NewRegistry(objectstore.NewMemoryObjectStore(), newTestMetaStore(t), testAccount)
+}
+
+func newTestMetaStore(t *testing.T) *ecr.KVMetaStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	return ecr.NewKVMetaStore(js)
 }
 
 func digestOf(b []byte) string {
@@ -69,7 +77,7 @@ func pushBlob(t *testing.T, reg *Registry, repo string, data []byte) string {
 }
 
 func TestBlobUpload_Monolithic_And_Head_Get(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("hello layer bytes")
 	dg := pushBlob(t, reg, "team/app", data)
 
@@ -84,20 +92,20 @@ func TestBlobUpload_Monolithic_And_Head_Get(t *testing.T) {
 }
 
 func TestBlobHead_NotFound(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodHead, "/v2/team/app/blobs/"+digestOf([]byte("absent")), nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }
 
 func TestBlobGet_MalformedDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/blobs/sha256:zzz", nil, nil)
 	assert.Equal(t, http.StatusBadRequest, w.Code)
 	assertCode(t, w, "DIGEST_INVALID")
 }
 
 func TestBlobUpload_Chunked(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	c1 := []byte("first-chunk-")
 	c2 := []byte("second-chunk")
@@ -124,7 +132,7 @@ func TestBlobUpload_Chunked(t *testing.T) {
 }
 
 func TestBlobUpload_DigestMismatch(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	w := do(reg, http.MethodPost, "/v2/r/repo/blobs/uploads/", nil, nil)
 	loc := w.Header().Get("Location")
@@ -134,7 +142,7 @@ func TestBlobUpload_DigestMismatch(t *testing.T) {
 }
 
 func TestBlobUpload_Cancel(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	seedRepo(t, reg, "r/repo")
 	w := do(reg, http.MethodPost, "/v2/r/repo/blobs/uploads/", nil, nil)
 	loc := w.Header().Get("Location")
@@ -161,7 +169,7 @@ func putManifestReferencing(t *testing.T, reg *Registry, repo, digest string) {
 }
 
 func TestBlobMount_HitAndMiss(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("shared-layer")
 	dg := pushBlob(t, reg, "team/source", data)
 	putManifestReferencing(t, reg, "team/source", dg)
@@ -185,7 +193,7 @@ func TestBlobMount_HitAndMiss(t *testing.T) {
 // as a plain upload start, matching the authorization middleware's
 // classification of the same request.
 func TestBlobMount_MissingFromFallsThroughToUpload(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	data := []byte("shared-layer")
 	dg := pushBlob(t, reg, "team/source", data)
 	putManifestReferencing(t, reg, "team/source", dg)
@@ -200,7 +208,7 @@ func TestBlobMount_MissingFromFallsThroughToUpload(t *testing.T) {
 // named source repository is rejected as a mount — proving digest existence
 // alone is not enough to mount cross-repo.
 func TestBlobMount_UnreferencedDigestFallsThrough(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	unreferenced := pushBlob(t, reg, "team/other", []byte("not-in-source"))
 	seedRepo(t, reg, "team/source")
 	seedRepo(t, reg, "team/dest")
@@ -214,7 +222,7 @@ func TestBlobMount_UnreferencedDigestFallsThrough(t *testing.T) {
 // repository that does not exist never mounts, and never distinguishes
 // itself from an unreferenced-digest miss (both fall through identically).
 func TestBlobMount_NonexistentSourceFallsThrough(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dg := pushBlob(t, reg, "team/elsewhere", []byte("some-bytes"))
 	seedRepo(t, reg, "team/dest")
 
@@ -224,7 +232,7 @@ func TestBlobMount_NonexistentSourceFallsThrough(t *testing.T) {
 }
 
 func TestManifest_PutGetHead_ImageManifest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 
 	cfg := []byte(`{"config":true}`)
@@ -255,7 +263,7 @@ func TestManifest_PutGetHead_ImageManifest(t *testing.T) {
 }
 
 func TestManifest_Put_MissingBlob(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	seedRepo(t, reg, repo)
 	manifest := fmt.Appendf(nil,
@@ -268,7 +276,7 @@ func TestManifest_Put_MissingBlob(t *testing.T) {
 }
 
 func TestManifest_Get_AcceptMismatch(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -285,7 +293,7 @@ func TestManifest_Get_AcceptMismatch(t *testing.T) {
 // acceptable manifest media type as its own Accept header line rather than one
 // comma-joined value. Negotiation must consider every line, not just the first.
 func TestManifest_Get_MultipleAcceptHeaders(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -305,7 +313,7 @@ func TestManifest_Get_MultipleAcceptHeaders(t *testing.T) {
 }
 
 func TestManifest_Get_Unknown(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/manifests/missing", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "MANIFEST_UNKNOWN")
@@ -314,7 +322,7 @@ func TestManifest_Get_Unknown(t *testing.T) {
 // TestStartUpload_RepoNotCreated asserts a blob upload to a repository that was
 // never created is rejected up front — ECR does not auto-create on push.
 func TestStartUpload_RepoNotCreated(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodPost, "/v2/team/ghost/blobs/uploads/", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 	assertCode(t, w, "NAME_UNKNOWN")
@@ -323,7 +331,7 @@ func TestStartUpload_RepoNotCreated(t *testing.T) {
 // TestPutManifest_RepoNotCreated asserts a manifest PUT to an uncreated
 // repository is rejected with NAME_UNKNOWN.
 func TestPutManifest_RepoNotCreated(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	manifest := fmt.Appendf(nil, `{"mediaType":"%s","layers":[]}`, mediaTypeDockerManifest)
 	w := do(reg, http.MethodPut, "/v2/team/ghost/manifests/v1", manifest,
 		map[string]string{"Content-Type": mediaTypeDockerManifest})
@@ -332,7 +340,7 @@ func TestPutManifest_RepoNotCreated(t *testing.T) {
 }
 
 func TestManifest_ImageIndex_ChildValidation(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 
 	// Push a child image manifest first.
@@ -372,7 +380,7 @@ func TestManifest_ImageIndex_ChildValidation(t *testing.T) {
 }
 
 func TestManifest_Delete_Tag(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -390,7 +398,7 @@ func TestManifest_Delete_Tag(t *testing.T) {
 // TestManifest_Delete_ByDigest proves delete-by-digest is no longer a no-op: it
 // removes the manifest record + object, and a subsequent GET is a 404.
 func TestManifest_Delete_ByDigest(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("cfg-bytes"))
 	manifest := fmt.Appendf(nil,
@@ -412,7 +420,7 @@ func TestManifest_Delete_ByDigest(t *testing.T) {
 }
 
 func TestTagsList(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	repo := "team/app"
 	cfg := pushBlob(t, reg, repo, []byte("c"))
 	manifest := fmt.Appendf(nil,
@@ -434,13 +442,13 @@ func TestTagsList(t *testing.T) {
 }
 
 func TestTagsList_UnknownRepo(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/ghost/tags/list", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }
 
 func TestCatalog(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	pushBlob(t, reg, "team/a", []byte("a"))
 	pushBlob(t, reg, "team/b", []byte("b"))
 
@@ -454,21 +462,21 @@ func TestCatalog(t *testing.T) {
 }
 
 func TestDispatch_InvalidRepoName(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/Team/App/tags/list", nil, nil)
 	assert.Equal(t, http.StatusBadRequest, w.Code)
 	assertCode(t, w, "NAME_INVALID")
 }
 
 func TestDispatch_RepeatedSeparatorRepoName(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	dg := pushBlob(t, reg, "team/a__b--c", []byte("layer"))
 	w := do(reg, http.MethodHead, "/v2/team/a__b--c/blobs/"+dg, nil, nil)
 	assert.Equal(t, http.StatusOK, w.Code)
 }
 
 func TestDispatch_UnknownPath(t *testing.T) {
-	reg := newTestRegistry()
+	reg := newTestRegistry(t)
 	w := do(reg, http.MethodGet, "/v2/team/app/frobnicate", nil, nil)
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }

spinifex/domains/ecr/registry/registry_upload_test.go

@@ -35,7 +35,7 @@ func (s *blobPutFailStore) PutObject(ctx context.Context, input *s3.PutObjectInp
 func TestFinishUpload_StoreFailure_CleansUp(t *testing.T) {
 	mem := objectstore.NewMemoryObjectStore()
 	store := &blobPutFailStore{MemoryObjectStore: mem}
-	meta := ecr.NewMemoryMetaStore()
+	meta := newTestMetaStore(t)
 	reg := NewRegistry(store, meta, testAccount)
 	seedRepo(t, reg, "team/app")
 
@@ -72,7 +72,7 @@ func awsString(s string) *string { return &s }
 // updates issued from the same observed revision cannot both commit. One wins,
 // the other gets ErrConflict — so a racing PATCH is rejected, never merged.
 func TestUploadCAS_SameRevisionSingleWinner(t *testing.T) {
-	meta := ecr.NewMemoryMetaStore()
+	meta := newTestMetaStore(t)
 	rev, err := meta.PutUpload(context.Background(), testAccount, "u-race", ecr.UploadState{RepoName: "r"})
 	require.NoError(t, err)
 
@@ -116,7 +116,7 @@ func TestUploadCAS_SameRevisionSingleWinner(t *testing.T) {
 // hash state and the recorded byte object never desynchronise.
 func TestPatchUpload_ConcurrentNoCorruption(t *testing.T) {
 	for iter := range 25 {
-		reg := newTestRegistry()
+		reg := newTestRegistry(t)
 		repo := "team/race"
 		seedRepo(t, reg, repo)
 

spinifex/domains/ecr/service_impl.go

@@ -16,8 +16,7 @@ type MetaServiceImpl struct {
 
 var _ MetaService = (*MetaServiceImpl)(nil)
 
-// NewMetaServiceImpl builds a MetaService over an explicit MetaStore (used by
-// daemon-side tests with a MemoryMetaStore).
+// NewMetaServiceImpl builds a MetaService over an explicit MetaStore.
 func NewMetaServiceImpl(store MetaStore) *MetaServiceImpl {
 	return &MetaServiceImpl{store: store}
 }

spinifex/domains/ecr/service_test.go

@@ -21,7 +21,7 @@ const svcTestAccount = "000000000000"
 // ErrNotFound/ErrConflict into the response flags rather than transport errors,
 // so they survive the AWS-error-code-only NATS reply.
 func TestMetaServiceImpl_FlagMapping(t *testing.T) {
-	svc := NewMetaServiceImpl(NewMemoryMetaStore())
+	svc := NewMetaServiceImpl(newTestKVMetaStore(t))
 
 	// Absent repo -> Found:false, nil error.
 	dr, err := svc.RepoDescribe(context.Background(), &RepoDescribeRequest{Repo: "ghost"}, svcTestAccount)

spinifex/domains/eks/cluster/delete_cluster_review_test.go

@@ -205,6 +205,8 @@ func TestS2_EKSLastNodegroupDeletedResumesTheClusterDelete(t *testing.T) {
 		ClusterName: aws.String("alpha"), NodegroupName: aws.String("ng1"),
 	}, testAccountID)
 	require.NoError(t, err)
+	_, err = f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
+	require.NoError(t, err)
 	f.svc.WaitTeardowns()
 
 	_, err = GetClusterMeta(t.Context(), f.kv, "alpha")

spinifex/domains/eks/cluster/delete_cluster_test.go

@@ -105,6 +105,9 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture {
 	require.NoError(t, err)
 	// Tight Ready-gating knobs so nodegroup launches resolve in tests without the
 	// production 10m timeout; tests drive meta.NodeCount to simulate the reconciler.
+	svc.noDeleteKick = true
+	svc.workersEndWait = 20 * time.Millisecond
+	svc.workersEndPoll = time.Millisecond
 	svc.nodegroupReadyTimeout = 50 * time.Millisecond
 	svc.nodegroupReadyPoll = 2 * time.Millisecond
 	// Tight worker-launch-retry knobs so a launch-failure retry test resolves

spinifex/domains/eks/cluster/eks_nodegroup_deleting_reaper.go

@@ -0,0 +1,284 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// A DELETING nodegroup becomes DELETE_FAILED once it has been deleting this
+// long and this many passes have failed. Vars for tests.
+var (
+	nodegroupDeleteFailAfter    = 30 * time.Minute
+	nodegroupDeleteFailAttempts = 5
+)
+
+// A pass waits this long, polling, for ended workers to be proven gone.
+const (
+	defaultWorkersEndWait = 90 * time.Second
+	defaultWorkersEndPoll = 3 * time.Second
+)
+
+// errWorkersEnding means workers were told to end but are not yet proven gone.
+var errWorkersEnding = errors.New("workers still ending")
+
+// EKSNodegroupDeletingReaper finishes recorded nodegroup deletes: it ends the
+// workers, removes the nodegroup's own instance profile and only then the
+// record, so a nodegroup is never gone while something it made still is.
+type EKSNodegroupDeletingReaper struct {
+	svc *EKSServiceImpl
+}
+
+var _ vm.Reaper = (*EKSNodegroupDeletingReaper)(nil)
+
+// NewNodegroupDeletingReaper builds the DELETING nodegroup converger.
+func (s *EKSServiceImpl) NewNodegroupDeletingReaper() *EKSNodegroupDeletingReaper {
+	return &EKSNodegroupDeletingReaper{svc: s}
+}
+
+func (r *EKSNodegroupDeletingReaper) Class() string         { return "eks-nodegroup-deleting" }
+func (r *EKSNodegroupDeletingReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
+
+// Sweep hands every due DELETING nodegroup to a background converger and
+// returns how many it started. The lease decides which node runs each one.
+func (r *EKSNodegroupDeletingReaper) Sweep(ctx context.Context) (int, error) {
+	if !r.svc.depsReadyForOrchestration() {
+		return 0, nil
+	}
+	js, err := jetstream.New(r.svc.deps.NATSConn)
+	if err != nil {
+		return 0, fmt.Errorf("jetstream: %w", err)
+	}
+	names, err := accountBucketNames(ctx, r.svc.deps.NATSConn)
+	if err != nil {
+		return 0, fmt.Errorf("enumerate account buckets: %w", err)
+	}
+
+	started := 0
+	for _, name := range names {
+		select {
+		case <-ctx.Done():
+			return started, ctx.Err()
+		default:
+		}
+		accountID := strings.TrimPrefix(name, KVBucketEKSAccountPrefix)
+		acctKV, err := js.KeyValue(ctx, name)
+		if err != nil {
+			continue
+		}
+		clusters, err := listClusterNames(ctx, acctKV)
+		if err != nil {
+			continue
+		}
+		for _, cluster := range clusters {
+			recs, err := ListNodegroupRecords(ctx, acctKV, cluster)
+			if err != nil {
+				continue
+			}
+			for _, rec := range recs {
+				if r.due(rec) && r.svc.startNodegroupDelete(accountID, acctKV, cluster, rec.Name) {
+					started++
+				}
+			}
+		}
+	}
+	return started, nil
+}
+
+// due reports whether rec is DELETING and past its retry backoff.
+func (r *EKSNodegroupDeletingReaper) due(rec *NodegroupRecord) bool {
+	if rec.Status != eks.NodegroupStatusDeleting {
+		return false
+	}
+	if rec.DeleteReapAttempts == 0 || rec.LastDeleteReapAttempt.IsZero() {
+		return true
+	}
+	return time.Since(rec.LastDeleteReapAttempt) >= deleteReapBackoff(deletingReapMinAge, rec.DeleteReapAttempts)
+}
+
+// startNodegroupDelete runs one converger pass in the background and reports
+// whether it started; a pass already running here or on another node refuses.
+func (s *EKSServiceImpl) startNodegroupDelete(accountID string, kv jetstream.KeyValue, cluster, ng string) bool {
+	release, _, ok := s.holdTeardown(accountID, cluster+"/nodegroup/"+ng, false)
+	if !ok {
+		return false
+	}
+	s.teardownWG.Go(func() {
+		defer release()
+		ctx, cancel := context.WithTimeout(s.bgCtx, teardownPassTimeout)
+		defer cancel()
+		if err := s.convergeNodegroupDeletion(ctx, accountID, kv, cluster, ng); err != nil {
+			slog.Warn("eks: nodegroup delete incomplete, the record stays DELETING for retry",
+				"cluster", cluster, "nodegroup", ng, "account", accountID, "err", err)
+		}
+	})
+	return true
+}
+
+// WaitNodegroupDeletes blocks until every converger this process started has
+// returned. Intended for tests.
+func (s *EKSServiceImpl) WaitNodegroupDeletes() { s.teardownWG.Wait() }
+
+// mutateNodegroup applies fn to the stored record under compare-and-swap. fn
+// returns false to leave the record unchanged.
+func (s *EKSServiceImpl) mutateNodegroup(ctx context.Context, kv jetstream.KeyValue, cluster, ng string, fn func(*NodegroupRecord) bool) (*NodegroupRecord, error) {
+	for range ngCASMaxRetries {
+		rec, rev, err := getNodegroupEntry(ctx, kv, cluster, ng)
+		if err != nil {
+			return nil, err
+		}
+		if !fn(rec) {
+			return rec, nil
+		}
+		ok, err := s.casPutNodegroup(ctx, kv, rec, rev)
+		if err != nil {
+			return nil, err
+		}
+		if ok {
+			return rec, nil
+		}
+	}
+	return nil, fmt.Errorf("nodegroup %s/%s: concurrent writers kept winning", cluster, ng)
+}
+
+// convergeNodegroupDeletion finishes one recorded delete. It acts only on a
+// record that is DELETING when re-read, and keeps the record on any failure.
+func (s *EKSServiceImpl) convergeNodegroupDeletion(ctx context.Context, accountID string, kv jetstream.KeyValue, cluster, ng string) error {
+	rec, err := s.mutateNodegroup(ctx, kv, cluster, ng, func(r *NodegroupRecord) bool {
+		if r.Status != eks.NodegroupStatusDeleting {
+			return false
+		}
+		r.DeleteReapAttempts++
+		r.LastDeleteReapAttempt = time.Now().UTC()
+		return true
+	})
+	if err != nil {
+		if errors.Is(err, ErrNodegroupNotFound) {
+			return nil
+		}
+		return err
+	}
+	if rec.Status != eks.NodegroupStatusDeleting {
+		return nil
+	}
+
+	if err := s.releaseNodegroupResources(ctx, accountID, kv, rec); err != nil {
+		if errors.Is(err, errWorkersEnding) {
+			s.noteWorkersEnding(ctx, kv, cluster, ng, err)
+			return err
+		}
+		s.noteNodegroupDeleteFailure(ctx, kv, rec, err)
+		return err
+	}
+	if err := DeleteNodegroupRecord(ctx, kv, cluster, ng); err != nil {
+		return err
+	}
+	// A cluster delete waits on its node groups; the last one going lets it run.
+	if meta, err := GetClusterMeta(ctx, kv, cluster); err == nil && meta.Status == ClusterStatusDeleting {
+		if rest, err := ListNodegroupRecords(ctx, kv, cluster); err == nil && len(rest) == 0 {
+			s.startTeardown(accountID, cluster)
+		}
+	}
+	return nil
+}
+
+// releaseNodegroupResources ends the workers, then removes the profile. The
+// profile and the recorded worker ids go only once every worker is proven
+// terminated or absent, since an accepted terminate is only a request.
+func (s *EKSServiceImpl) releaseNodegroupResources(ctx context.Context, accountID string, kv jetstream.KeyValue, rec *NodegroupRecord) error {
+	if len(rec.InstanceIDs) > 0 {
+		if err := s.deps.Worker.TerminateWorkerInstances(ctx, rec.InstanceIDs, accountID); err != nil {
+			return fmt.Errorf("terminate workers: %w", err)
+		}
+		if err := s.awaitWorkersGone(ctx, rec.InstanceIDs, accountID); err != nil {
+			return err
+		}
+		if _, err := s.mutateNodegroup(ctx, kv, rec.ClusterName, rec.Name, func(r *NodegroupRecord) bool {
+			r.InstanceIDs = nil
+			return true
+		}); err != nil {
+			return fmt.Errorf("record workers ended: %w", err)
+		}
+	}
+	return s.deleteNodegroupProfile(ctx, accountID, kv, rec)
+}
+
+// awaitWorkersGone polls until no worker is live, up to the end wait,
+// and returns errWorkersEnding if some still are.
+func (s *EKSServiceImpl) awaitWorkersGone(ctx context.Context, ids []string, accountID string) error {
+	wait, poll := s.workersEndWait, s.workersEndPoll
+	if wait <= 0 {
+		wait = defaultWorkersEndWait
+	}
+	if poll <= 0 {
+		poll = defaultWorkersEndPoll
+	}
+	deadline := time.Now().Add(wait)
+	for {
+		live, err := s.deps.Worker.LiveWorkerInstances(ctx, ids, accountID)
+		if err != nil {
+			return fmt.Errorf("confirm workers ended: %w", err)
+		}
+		if len(live) == 0 {
+			return nil
+		}
+		if !time.Now().Before(deadline) {
+			return fmt.Errorf("%w: %s", errWorkersEnding, strings.Join(live, ","))
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(poll):
+		}
+	}
+}
+
+// noteWorkersEnding handles workers that are not yet gone. Early on that is no
+// failed attempt: the next sweep checks again without backoff. Once the group
+// has been deleting past nodegroupDeleteFailAfter it settles DELETE_FAILED
+// naming the instances, keeping the profile and record for a customer retry.
+func (s *EKSServiceImpl) noteWorkersEnding(ctx context.Context, kv jetstream.KeyValue, cluster, ng string, cause error) {
+	_, err := s.mutateNodegroup(ctx, kv, cluster, ng, func(r *NodegroupRecord) bool {
+		if r.Status != eks.NodegroupStatusDeleting {
+			return false
+		}
+		if time.Since(r.DeletingSince) >= nodegroupDeleteFailAfter {
+			r.Status = eks.NodegroupStatusDeleteFailed
+			r.StatusReason = "delete: " + cause.Error()
+			r.ModifiedAt = time.Now().UTC()
+			return true
+		}
+		r.DeleteReapAttempts--
+		r.LastDeleteReapAttempt = time.Time{}
+		return true
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "eks: record workers still ending", "cluster", cluster, "nodegroup", ng, "err", err)
+	}
+}
+
+// noteNodegroupDeleteFailure records why a pass failed and, once the group has
+// been deleting long enough and failed often enough, settles it DELETE_FAILED.
+func (s *EKSServiceImpl) noteNodegroupDeleteFailure(ctx context.Context, kv jetstream.KeyValue, rec *NodegroupRecord, cause error) {
+	_, err := s.mutateNodegroup(ctx, kv, rec.ClusterName, rec.Name, func(r *NodegroupRecord) bool {
+		if r.Status != eks.NodegroupStatusDeleting {
+			return false
+		}
+		r.StatusReason = "delete: " + cause.Error()
+		r.ModifiedAt = time.Now().UTC()
+		if r.DeleteReapAttempts >= nodegroupDeleteFailAttempts && time.Since(r.DeletingSince) >= nodegroupDeleteFailAfter {
+			r.Status = eks.NodegroupStatusDeleteFailed
+		}
+		return true
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "eks: record nodegroup delete failure", "cluster", rec.ClusterName, "nodegroup", rec.Name, "err", err)
+	}
+}

spinifex/domains/eks/cluster/nodegroup.go

@@ -379,6 +379,12 @@ func (s *EKSServiceImpl) createNodegroup(ctx context.Context, acctKV jetstream.K
 		ModifiedAt:     now,
 	}
 
+	// Recorded before the claim, so a name is durable before any worker launches.
+	if rec.NodeRole != "" {
+		rec.InstanceProfileID = uuid.NewV4().String()
+		rec.InstanceProfileName = nodegroupProfilePrefix + rec.InstanceProfileID
+	}
+
 	// Atomically claim the record; concurrent/duplicate CreateNodegroup requests lose here.
 	owned, err := ClaimNodegroupRecord(ctx, acctKV, rec)
 	if err != nil {
@@ -716,7 +722,7 @@ func (s *EKSServiceImpl) launchOneWorker(ctx context.Context, rec *NodegroupReco
 	// Attach the node role's instance profile so IMDS serves the role to the ECR
 	// credential provider; without it worker pulls from the internal ECR get 401.
 	if s.iamEnsurer() != nil && rec.NodeRole != "" {
-		profileARN, perr := s.ensureNodeInstanceProfile(accountID, rec.NodeRole)
+		profileARN, perr := s.nodeProfileForLaunch(accountID, rec)
 		if perr != nil {
 			return "", fmt.Errorf("ensure node instance profile: %w", perr)
 		}
@@ -855,14 +861,10 @@ func gatewayHostIP(gatewayURL string) string {
 	return host
 }
 
-// ensureNodeInstanceProfile guarantees an IAM instance profile named for the
-// nodegroup's node role exists and carries that role, returning its ARN. Workers
-// launch with this profile so IMDS serves the node role to the ECR credential
-// provider, mirroring the implicit instance profile real EKS creates for a node
-// role. Idempotent: concurrent worker launches converge on the same profile.
-func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string) (string, error) {
-	// Resolution compares the supplied ARN against the stored one, so an invented
-	// path cannot bind a profile to the role its trailing name happens to match.
+// resolveNodeRoleName returns the name of the stored role nodeRoleARN names.
+// Resolution compares the supplied ARN against the stored one, so an invented
+// path cannot bind a profile to the role its trailing name happens to match.
+func (s *EKSServiceImpl) resolveNodeRoleName(accountID, nodeRoleARN string) (string, error) {
 	_, roleName, err := auth.ResolveRoleARN(nodeRoleARN, func(account, name string) (string, error) {
 		// The profile is created under the caller's account, so a role from
 		// another account would silently bind a name that account does not own.
@@ -881,6 +883,19 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 	if err != nil {
 		return "", fmt.Errorf("node role ARN %q: %w", nodeRoleARN, err)
 	}
+	return roleName, nil
+}
+
+// ensureNodeInstanceProfile guarantees an IAM instance profile named for the
+// nodegroup's node role exists and carries that role, returning its ARN. Workers
+// launch with this profile so IMDS serves the node role to the ECR credential
+// provider, mirroring the implicit instance profile real EKS creates for a node
+// role. Idempotent: concurrent worker launches converge on the same profile.
+func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string) (string, error) {
+	roleName, err := s.resolveNodeRoleName(accountID, nodeRoleARN)
+	if err != nil {
+		return "", err
+	}
 	profileName := roleName
 
 	out, err := s.iamEnsurer().GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{
@@ -1018,6 +1033,9 @@ func (s *EKSServiceImpl) reconcileNodegroup(ctx context.Context, acctKV jetstrea
 			}
 			return nil, err
 		}
+		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusDeleteFailed {
+			return nil, errors.New(awserrors.ErrorEKSResourceInUse)
+		}
 		applyScalingUpdate(rec, scaling)
 		if labels != nil {
 			rec.Labels = applyLabelUpdate(rec.Labels, labels)
@@ -1227,33 +1245,49 @@ func (s *EKSServiceImpl) deleteNodegroup(ctx context.Context, acctKV jetstream.K
 	if err := requireCluster(ctx, acctKV, cluster); err != nil {
 		return nil, err
 	}
-	rec, err := GetNodegroupRecord(ctx, acctKV, cluster, ng)
+	rec, err := s.markNodegroupDeleting(ctx, acctKV, cluster, ng)
 	if err != nil {
-		if errors.Is(err, ErrNodegroupNotFound) {
-			return nil, nodegroupNotFound(ng)
-		}
 		return nil, err
 	}
-
-	if len(rec.InstanceIDs) > 0 {
-		if err := s.deps.Worker.TerminateWorkerInstances(ctx, rec.InstanceIDs, accountID); err != nil {
-			return nil, fmt.Errorf("terminate workers: %w", err)
-		}
+	if !s.noDeleteKick {
+		s.startNodegroupDelete(accountID, acctKV, cluster, ng)
 	}
-	if err := DeleteNodegroupRecord(ctx, acctKV, cluster, ng); err != nil {
-		return nil, err
-	}
-	// A cluster delete waits on its node groups; the last one going lets it run.
-	if meta, err := GetClusterMeta(ctx, acctKV, cluster); err == nil && meta.Status == ClusterStatusDeleting {
-		if rest, err := ListNodegroupRecords(ctx, acctKV, cluster); err == nil && len(rest) == 0 {
-			s.startTeardown(accountID, cluster)
-		}
-	}
-
-	rec.Status = eks.NodegroupStatusDeleting
 	return &eks.DeleteNodegroupOutput{Nodegroup: nodegroupRecordToAWS(rec)}, nil
 }
 
+// markNodegroupDeleting records DELETING and returns. The workers, the profile
+// and the record itself are the nodegroup reaper's to remove, so the request
+// never waits on work that scales with the group (a launching group is refused).
+func (s *EKSServiceImpl) markNodegroupDeleting(ctx context.Context, kv jetstream.KeyValue, cluster, ng string) (*NodegroupRecord, error) {
+	for range ngCASMaxRetries {
+		rec, rev, err := getNodegroupEntry(ctx, kv, cluster, ng)
+		if err != nil {
+			if errors.Is(err, ErrNodegroupNotFound) {
+				return nil, nodegroupNotFound(ng)
+			}
+			return nil, err
+		}
+		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusCreating {
+			return nil, errors.New(awserrors.ErrorEKSResourceInUse)
+		}
+		now := time.Now().UTC()
+		rec.Status = eks.NodegroupStatusDeleting
+		rec.StatusReason = ""
+		rec.DeletingSince = now
+		rec.DeleteReapAttempts = 0
+		rec.LastDeleteReapAttempt = time.Time{}
+		rec.ModifiedAt = now
+		ok, err := s.casPutNodegroup(ctx, kv, rec, rev)
+		if err != nil {
+			return nil, err
+		}
+		if ok {
+			return rec, nil
+		}
+	}
+	return nil, errors.New(awserrors.ErrorServerInternal)
+}
+
 // decryptNodeToken reads + decrypts the cluster's K3s join token from KV.
 func (s *EKSServiceImpl) decryptNodeToken(ctx context.Context, kv jetstream.KeyValue, cluster string) (string, error) {
 	if kv == nil {
@@ -1496,8 +1530,12 @@ func nodegroupRecordToAWS(rec *NodegroupRecord) *eks.Nodegroup {
 	// wrong; a nil slice here would marshal the key away entirely.
 	issues := make([]*eks.Issue, 0, 1)
 	if rec.StatusReason != "" {
+		code := eks.NodegroupIssueCodeNodeCreationFailure
+		if rec.Status == eks.NodegroupStatusDeleting || rec.Status == eks.NodegroupStatusDeleteFailed {
+			code = eks.NodegroupIssueCodeNodeTerminationFailure
+		}
 		issues = append(issues, &eks.Issue{
-			Code:        aws.String(eks.NodegroupIssueCodeNodeCreationFailure),
+			Code:        aws.String(code),
 			Message:     aws.String(rec.StatusReason),
 			ResourceIds: aws.StringSlice(rec.InstanceIDs),
 		})

spinifex/domains/eks/cluster/nodegroup_delete_failure_test.go

@@ -0,0 +1,255 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func (f *fakeWorkerLauncher) terminateCallCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return len(f.terminateCalls)
+}
+
+func requireNodegroupStatus(t *testing.T, f *eksServiceFixture, cluster, ng, want string) *NodegroupRecord {
+	t.Helper()
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, cluster, ng)
+	require.NoError(t, err)
+	require.Equal(t, want, rec.Status)
+	return rec
+}
+
+func TestNodegroupDeletingReaper_SweepWithoutOrchestrationDepsStartsNothing(t *testing.T) {
+	t.Parallel()
+	n, err := (&EKSServiceImpl{}).NewNodegroupDeletingReaper().Sweep(t.Context())
+	require.NoError(t, err)
+	assert.Zero(t, n)
+}
+
+func TestNodegroupDeletingReaper_CancelledSweepLeavesTheDeleteRecorded(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	n, err := f.svc.NewNodegroupDeletingReaper().Sweep(ctx)
+	f.svc.WaitNodegroupDeletes()
+
+	require.ErrorIs(t, err, context.Canceled)
+	assert.Zero(t, n)
+	requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusDeleting)
+}
+
+func TestNodegroupDeletingReaper_OnlyDeletingGroupsPastBackoffAreStarted(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "active", 1)
+	createReadyNodegroup(t, f, "c1", "backoff", 1)
+	_, err := deleteNodegroupCall(f, "c1", "backoff")
+	require.NoError(t, err)
+	rec := requireNodegroupStatus(t, f, "c1", "backoff", eks.NodegroupStatusDeleting)
+	rec.DeleteReapAttempts = 3
+	rec.LastDeleteReapAttempt = time.Now().UTC()
+	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, rec))
+
+	assert.Zero(t, sweepNodegroups(t, f), "an ACTIVE group and a DELETING one inside its backoff are not due")
+	requireNodegroupStatus(t, f, "c1", "active", eks.NodegroupStatusActive)
+	assert.Zero(t, f.worker.terminateCallCount())
+}
+
+func TestConvergeNodegroupDeletion_MissingOrNotDeletingRecordIsLeftAlone(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+
+	require.NoError(t, f.svc.convergeNodegroupDeletion(t.Context(), testAccountID, f.kv, "c1", "never-existed"))
+	require.NoError(t, f.svc.convergeNodegroupDeletion(t.Context(), testAccountID, f.kv, "c1", "ng1"))
+
+	requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusActive)
+	assert.Zero(t, f.worker.terminateCallCount(), "a group nobody asked to delete keeps its workers")
+	assert.Contains(t, fixtureIAM(t, f).Profiles, mustProfileName(t, f, "c1", "ng1"))
+}
+
+func mustProfileName(t *testing.T, f *eksServiceFixture, cluster, ng string) string {
+	t.Helper()
+	rec, _ := ownedProfile(t, f, cluster, ng)
+	return rec.InstanceProfileName
+}
+
+func TestAwaitWorkersGone_CancelledWhileWorkersEndingReturnsTheContextError(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	f.worker.stillLive = []string{"i-slow"}
+	f.svc.workersEndWait = time.Hour
+	f.svc.workersEndPoll = time.Hour
+	ctx, cancel := context.WithCancel(t.Context())
+	time.AfterFunc(50*time.Millisecond, cancel)
+
+	err := f.svc.awaitWorkersGone(ctx, []string{"i-slow"}, testAccountID)
+
+	require.ErrorIs(t, err, context.Canceled)
+	assert.NotErrorIs(t, err, errWorkersEnding, "a cancelled wait is not a verdict on the workers")
+}
+
+func TestDeleteNodegroup_IAMReadFailureKeepsGroupDeletingAndProfileIntact(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	m.GetInstanceProfileErr = errors.New("iam read timed out")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	cur := requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusDeleting)
+	assert.Contains(t, cur.StatusReason, "get instance profile", "the failed read is recorded for the customer")
+	assert.Contains(t, m.Profiles, rec.InstanceProfileName, "an unreadable profile is not a missing one")
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+
+	m.GetInstanceProfileErr = nil
+	backdateNodegroupReap(t, f, "c1", "ng1")
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+}
+
+func TestDeleteNodegroup_ProfileAlreadyGoneStillFinishesTheGroup(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	delete(m.Profiles, rec.InstanceProfileName)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.Zero(t, m.RemoveRoleCalls)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+}
+
+func TestDeleteNodegroup_ProfileVanishingMidDeleteStillFinishesTheGroup(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	m := fixtureIAM(t, f)
+	m.DeleteInstanceProfileErr = awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile gone.")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.Equal(t, 1, m.RemoveRoleCalls)
+}
+
+func TestDeleteNodegroup_DeleteProfileFailureKeepsGroupDeletingUntilItClears(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	m.DeleteInstanceProfileErr = errors.New("iam unavailable")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+	cur := requireNodegroupStatus(t, f, "c1", "ng1", eks.NodegroupStatusDeleting)
+	assert.Contains(t, cur.StatusReason, "delete instance profile")
+	assert.Empty(t, cur.InstanceIDs, "workers proven gone are not terminated again")
+
+	m.DeleteInstanceProfileErr = nil
+	backdateNodegroupReap(t, f, "c1", "ng1")
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+}
+
+func TestEnsureNodegroupProfile_RecreatesAMissingRecordedProfileAsItsOwn(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	delete(m.Profiles, rec.InstanceProfileName)
+
+	arn, err := f.svc.ensureNodegroupProfile(testAccountID, rec)
+
+	require.NoError(t, err)
+	assert.Contains(t, arn, rec.InstanceProfileName)
+	got := m.Profiles[rec.InstanceProfileName]
+	require.NotNil(t, got)
+	assert.True(t, profileOwnedBy(got, rec), "a re-created profile carries this incarnation's tags")
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(got))
+}
+
+func TestEnsureNodegroupProfile_RefusesAProfileThatIsNotThisGroups(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, p := ownedProfile(t, f, "c1", "ng1")
+	p.Tags = nil
+	callsBefore := fixtureIAM(t, f).AddRoleToInstanceProfileCalls
+
+	_, err := f.svc.ensureNodegroupProfile(testAccountID, rec)
+
+	require.ErrorContains(t, err, "is not this nodegroup's")
+	assert.Equal(t, callsBefore, fixtureIAM(t, f).AddRoleToInstanceProfileCalls, "no role is attached to a profile EKS does not own")
+}
+
+func TestEnsureNodegroupProfile_IAMFailuresSurfaceWithoutHalfWork(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+
+	m.GetInstanceProfileErr = errors.New("iam read failed")
+	_, err := f.svc.ensureNodegroupProfile(testAccountID, rec)
+	require.ErrorContains(t, err, "get instance profile")
+	m.GetInstanceProfileErr = nil
+
+	delete(m.Profiles, rec.InstanceProfileName)
+	m.CreateInstanceProfileErr = errors.New("iam write failed")
+	_, err = f.svc.ensureNodegroupProfile(testAccountID, rec)
+	require.ErrorContains(t, err, "create instance profile")
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+}
+
+func TestMarkNodegroupDeleting_UnknownGroupIsResourceNotFound(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+
+	_, err := f.svc.markNodegroupDeleting(t.Context(), f.kv, "c1", "ghost")
+
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}

spinifex/domains/eks/cluster/nodegroup_delete_test.go

@@ -0,0 +1,662 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/aws/aws-sdk-go/service/iam"
+	iammock "github.com/mulgadc/spinifex/spinifex/domains/iam/mock"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	profileClusterTag   = "spinifex:eks-cluster"
+	profileNodegroupTag = "spinifex:eks-nodegroup"
+	fixtureNodeRoleName = "eks-node"
+)
+
+func fixtureIAM(t *testing.T, f *eksServiceFixture) *iammock.SystemInstanceRoleEnsurer {
+	t.Helper()
+	m, ok := f.svc.deps.IAM.(*iammock.SystemInstanceRoleEnsurer)
+	require.True(t, ok, "the fixture IAM must be the in-memory mock")
+	return m
+}
+
+// createReadyNodegroup creates a nodegroup whose n workers register Ready.
+func createReadyNodegroup(t *testing.T, f *eksServiceFixture, cluster, ng string, n int64) {
+	t.Helper()
+	markWorkersReady(t, f, cluster, ng, int(n))
+	_, err := f.svc.CreateNodegroup(context.Background(), createNGInput(cluster, ng, n), testAccountID)
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+}
+
+// createUnreadyNodegroup creates a nodegroup that never registers, so it ends
+// CREATE_FAILED holding the workers it launched.
+func createUnreadyNodegroup(t *testing.T, f *eksServiceFixture, cluster, ng string, n int64) {
+	t.Helper()
+	_, err := f.svc.CreateNodegroup(context.Background(), createNGInput(cluster, ng, n), testAccountID)
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+}
+
+func deleteNodegroupCall(f *eksServiceFixture, cluster, ng string) (*eks.DeleteNodegroupOutput, error) {
+	return f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
+		ClusterName: aws.String(cluster), NodegroupName: aws.String(ng),
+	}, testAccountID)
+}
+
+func describeNodegroupCall(f *eksServiceFixture, cluster, ng string) (*eks.DescribeNodegroupOutput, error) {
+	return f.svc.DescribeNodegroup(context.Background(), &eks.DescribeNodegroupInput{
+		ClusterName: aws.String(cluster), NodegroupName: aws.String(ng),
+	}, testAccountID)
+}
+
+func sweepNodegroups(t *testing.T, f *eksServiceFixture) int {
+	t.Helper()
+	n, err := f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
+	require.NoError(t, err)
+	f.svc.WaitNodegroupDeletes()
+	return n
+}
+
+// backdateNodegroupReap moves the last reap pass far enough back that any
+// retry backoff has elapsed.
+func backdateNodegroupReap(t *testing.T, f *eksServiceFixture, cluster, ng string) {
+	t.Helper()
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, cluster, ng)
+	require.NoError(t, err)
+	rec.LastDeleteReapAttempt = time.Now().UTC().Add(-24 * time.Hour)
+	rec.DeletingSince = time.Now().UTC().Add(-24 * time.Hour)
+	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, rec))
+}
+
+func requireErrorCode(t *testing.T, err error, want string) {
+	t.Helper()
+	require.Error(t, err)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok, "error %v carries no AWS code", err)
+	assert.Equal(t, want, code)
+}
+
+func profileTag(p *iam.InstanceProfile, key string) string {
+	for _, tg := range p.Tags {
+		if aws.StringValue(tg.Key) == key {
+			return aws.StringValue(tg.Value)
+		}
+	}
+	return ""
+}
+
+func profileHasTagValue(p *iam.InstanceProfile, value string) bool {
+	for _, tg := range p.Tags {
+		if aws.StringValue(tg.Value) == value {
+			return true
+		}
+	}
+	return false
+}
+
+func profileRoleNames(p *iam.InstanceProfile) []string {
+	var names []string
+	for _, r := range p.Roles {
+		names = append(names, aws.StringValue(r.RoleName))
+	}
+	return names
+}
+
+// ownedProfile returns the nodegroup's record and the instance profile it
+// names, failing the test if either is missing.
+func ownedProfile(t *testing.T, f *eksServiceFixture, cluster, ng string) (*NodegroupRecord, *iam.InstanceProfile) {
+	t.Helper()
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, cluster, ng)
+	require.NoError(t, err)
+	require.NotEmpty(t, rec.InstanceProfileName, "the record must name the profile it owns")
+	require.NotEmpty(t, rec.InstanceProfileID, "the record must carry the per-incarnation id")
+	p, ok := fixtureIAM(t, f).Profiles[rec.InstanceProfileName]
+	require.True(t, ok, "profile %q must exist", rec.InstanceProfileName)
+	return rec, p
+}
+
+func seedCustomerProfile(f *eksServiceFixture, t *testing.T, name, role string, tags map[string]string) {
+	t.Helper()
+	p := &iam.InstanceProfile{
+		InstanceProfileName: aws.String(name),
+		Arn:                 aws.String("arn:aws:iam::" + testAccountID + ":instance-profile/" + name),
+	}
+	if role != "" {
+		p.Roles = []*iam.Role{{RoleName: aws.String(role)}}
+	}
+	for k, v := range tags {
+		p.Tags = append(p.Tags, &iam.Tag{Key: aws.String(k), Value: aws.String(v)})
+	}
+	fixtureIAM(t, f).Profiles[name] = p
+}
+
+func TestDeleteNodegroup_ReturnsDeletingWithoutTerminatingInRequest(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+
+	out, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleting, aws.StringValue(out.Nodegroup.Status))
+
+	assert.Zero(t, f.worker.terminatedCount(), "the request must not wait for workers to end")
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err, "the record stays until the reaper has finished")
+	assert.Equal(t, eks.NodegroupStatusDeleting, rec.Status)
+
+	desc, err := describeNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleting, aws.StringValue(desc.Nodegroup.Status))
+}
+
+func TestDeleteNodegroup_ReaperEndsWorkersThenRemovesGroup(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, 1, sweepNodegroups(t, f))
+
+	require.Len(t, f.worker.terminateCalls, 1)
+	assert.ElementsMatch(t, rec.InstanceIDs, f.worker.terminateCalls[0])
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	_, ok := fixtureIAM(t, f).Profiles[rec.InstanceProfileName]
+	assert.False(t, ok, "the group is not gone until its profile is")
+}
+
+func TestDeleteNodegroup_RepeatedDeleteOnDeletingGroupIsResourceInUse(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	_, err = deleteNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
+}
+
+func TestDeleteNodegroup_DeletingGroupRefusesUpdates(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	_, err = f.svc.UpdateNodegroupConfig(context.Background(), &eks.UpdateNodegroupConfigInput{
+		ClusterName:   aws.String("c1"),
+		NodegroupName: aws.String("ng1"),
+		ScalingConfig: &eks.NodegroupScalingConfig{MinSize: aws.Int64(1), MaxSize: aws.Int64(3), DesiredSize: aws.Int64(3)},
+	}, testAccountID)
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
+	assert.Equal(t, 1, f.worker.runCount(), "a DELETING group must not launch workers")
+}
+
+func TestDeleteNodegroup_SameNameCreateWhileDeletingIsResourceInUse(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	_, err = f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
+}
+
+func TestDeleteNodegroup_DeleteClusterRefusedWhileGroupDeleting(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	_, err = f.svc.DeleteCluster(context.Background(), &eks.DeleteClusterInput{Name: aws.String("c1")}, testAccountID)
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceInUse)
+}
+
+func TestDeleteNodegroup_BootReclaimLeavesDeletingGroupToReaper(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{
+		ClusterName: "c1", Name: "ng1", Status: eks.NodegroupStatusDeleting,
+		InstanceIDs: []string{"i-w1", "i-w2"}, NodeRole: fixtureNodeRoleARN,
+	}))
+
+	f.svc.reclaimOrphanedNodegroups(t.Context(), testAccountID, f.kv, "c1")
+
+	assert.Zero(t, f.worker.terminatedCount())
+	rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleting, rec.Status)
+	assert.Equal(t, []string{"i-w1", "i-w2"}, rec.InstanceIDs)
+}
+
+func TestDeleteNodegroup_RepeatedReaperFailureEndsDeleteFailedWithHealthIssue(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	f.worker.termErr = errors.New("terminate refused")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	var rec *NodegroupRecord
+	for range 30 {
+		sweepNodegroups(t, f)
+		rec, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+		require.NoError(t, err, "a failing delete must leave the record, never drop it")
+		if rec.Status == eks.NodegroupStatusDeleteFailed {
+			break
+		}
+		backdateNodegroupReap(t, f, "c1", "ng1")
+	}
+	require.Equal(t, eks.NodegroupStatusDeleteFailed, rec.Status)
+
+	desc, err := describeNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	require.NotEmpty(t, desc.Nodegroup.Health.Issues, "DELETE_FAILED must say why")
+	assert.Equal(t, eks.NodegroupIssueCodeNodeTerminationFailure, aws.StringValue(desc.Nodegroup.Health.Issues[0].Code))
+
+	// The customer retries the delete once the cause is gone.
+	f.worker.termErr = nil
+	_, err = deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}
+
+func TestCreateNodegroup_OwnsATaggedProfileRecordedBeforeFirstLaunch(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	var seenAtLaunch string
+	f.worker.beforeRun = func(in *ec2.RunInstancesInput) {
+		if rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1"); err == nil && seenAtLaunch == "" {
+			seenAtLaunch = rec.InstanceProfileName
+		}
+	}
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+
+	rec, p := ownedProfile(t, f, "c1", "ng1")
+	assert.NotEqual(t, fixtureNodeRoleName, rec.InstanceProfileName, "the profile is the group's own, not the role-named one")
+	assert.Equal(t, rec.InstanceProfileName, seenAtLaunch, "the name must be durable before any worker launches")
+	assert.Equal(t, "c1", profileTag(p, profileClusterTag))
+	assert.Equal(t, "ng1", profileTag(p, profileNodegroupTag))
+	assert.True(t, profileHasTagValue(p, rec.InstanceProfileID), "a tag must carry the incarnation id")
+	for _, tg := range p.Tags {
+		assert.True(t, strings.HasPrefix(aws.StringValue(tg.Key), "spinifex:"), "tag %q: aws: keys are reserved, ownership tags must be spinifex:", aws.StringValue(tg.Key))
+	}
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(p))
+
+	require.Equal(t, 2, f.worker.runCount())
+	for _, in := range f.worker.runCalls {
+		require.NotNil(t, in.IamInstanceProfile)
+		assert.True(t, strings.HasSuffix(aws.StringValue(in.IamInstanceProfile.Arn), "/"+rec.InstanceProfileName),
+			"every worker launches with the group's own profile, got %s", aws.StringValue(in.IamInstanceProfile.Arn))
+	}
+}
+
+func TestCreateNodegroup_TwoGroupsOnOneRoleGetTwoProfiles(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createUnreadyNodegroup(t, f, "c1", "ng-a", 1)
+	createUnreadyNodegroup(t, f, "c1", "ng-b", 1)
+
+	recA, pA := ownedProfile(t, f, "c1", "ng-a")
+	recB, pB := ownedProfile(t, f, "c1", "ng-b")
+	assert.NotEqual(t, recA.InstanceProfileName, recB.InstanceProfileName)
+	assert.NotEqual(t, recA.InstanceProfileID, recB.InstanceProfileID)
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(pA))
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(pB))
+}
+
+func TestCreateNodegroup_RecreatedGroupOfSameNameGetsANewProfile(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createUnreadyNodegroup(t, f, "c1", "ng1", 1)
+	first, _ := ownedProfile(t, f, "c1", "ng1")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	createUnreadyNodegroup(t, f, "c1", "ng1", 1)
+	second, _ := ownedProfile(t, f, "c1", "ng1")
+	assert.NotEqual(t, first.InstanceProfileName, second.InstanceProfileName)
+	assert.NotEqual(t, first.InstanceProfileID, second.InstanceProfileID)
+	_, stillThere := fixtureIAM(t, f).Profiles[first.InstanceProfileName]
+	assert.False(t, stillThere, "the first incarnation's profile must have gone with it")
+}
+
+func TestDeleteNodegroup_AfterCreateFailedRemovesProfileAndGroup(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createUnreadyNodegroup(t, f, "c1", "ng1", 2)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	require.Equal(t, eks.NodegroupStatusCreateFailed, rec.Status)
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	_, ok := fixtureIAM(t, f).Profiles[rec.InstanceProfileName]
+	assert.False(t, ok)
+	assert.Equal(t, 2, f.worker.terminatedCount())
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}
+
+func TestDeleteNodegroup_ProfileSurvivesWhileWorkersAreLive(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	f.worker.termErr = errors.New("terminate refused")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	m := fixtureIAM(t, f)
+	assert.Contains(t, m.Profiles, rec.InstanceProfileName, "a profile in use by live workers must not be touched")
+	assert.Zero(t, m.RemoveRoleCalls)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+}
+
+func TestDeleteNodegroup_PartialProfileDeleteConvergesOnRetry(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	m.RemoveRoleFromInstanceProfileErr = errors.New("iam unavailable")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+	still, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err, "the group is not gone while its profile is")
+	assert.Equal(t, eks.NodegroupStatusDeleting, still.Status)
+
+	m.RemoveRoleFromInstanceProfileErr = nil
+	backdateNodegroupReap(t, f, "c1", "ng1")
+	sweepNodegroups(t, f)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}
+
+func TestDeleteNodegroup_CustomerRoleNamedProfileSurvives(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	seedCustomerProfile(f, t, fixtureNodeRoleName, fixtureNodeRoleName, nil)
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	m := fixtureIAM(t, f)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName, "the group's own profile goes")
+	require.Contains(t, m.Profiles, fixtureNodeRoleName, "a customer's profile is never EKS's to delete")
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[fixtureNodeRoleName]))
+	assert.Equal(t, 1, m.DeleteInstanceProfileCalls)
+}
+
+func TestDeleteNodegroup_RecordedProfileMissingOwnershipTagSurvives(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	m := fixtureIAM(t, f)
+	_, err := m.UntagInstanceProfile(testAccountID, &iam.UntagInstanceProfileInput{
+		InstanceProfileName: aws.String(rec.InstanceProfileName),
+		TagKeys:             aws.StringSlice([]string{profileClusterTag}),
+	})
+	require.NoError(t, err)
+
+	_, err = deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	require.Contains(t, m.Profiles, rec.InstanceProfileName)
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[rec.InstanceProfileName]))
+	assert.Zero(t, m.RemoveRoleCalls)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+}
+
+func TestDeleteNodegroup_OldLayoutGroupDeletesAndLeavesRoleNamedProfile(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	seedCustomerProfile(f, t, fixtureNodeRoleName, fixtureNodeRoleName, nil)
+	require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{
+		ClusterName: "c1", Name: "old", Status: eks.NodegroupStatusActive,
+		InstanceIDs: []string{"i-old1"}, NodeRole: fixtureNodeRoleARN,
+	}))
+
+	_, err := deleteNodegroupCall(f, "c1", "old")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	require.Len(t, f.worker.terminateCalls, 1)
+	assert.Equal(t, []string{"i-old1"}, f.worker.terminateCalls[0])
+	_, err = describeNodegroupCall(f, "c1", "old")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	m := fixtureIAM(t, f)
+	require.Contains(t, m.Profiles, fixtureNodeRoleName)
+	assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[fixtureNodeRoleName]))
+	assert.Zero(t, m.RemoveRoleCalls)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+}
+
+// No path deletes a profile unless it is recorded, tagged for this
+// incarnation, holds the record's node role and is named by no other record.
+func TestInvariant_NodegroupDeleteNeverRemovesAProfileItDoesNotOwn(t *testing.T) {
+	t.Parallel()
+	ownership := map[string]string{profileClusterTag: "c1", profileNodegroupTag: "ng1"}
+
+	t.Run("customer pre-creates a profile at a guessed name", func(t *testing.T) {
+		t.Parallel()
+		f := newEKSServiceFixture(t)
+		seedActiveClusterWithToken(t, f, "c1")
+		guesses := []string{
+			"eks-" + arn.EKSNodegroupDiscriminator(testAccountID, "c1", "ng1"),
+			"eks-c1-ng1",
+			"c1-ng1",
+			fixtureNodeRoleName,
+		}
+		for _, name := range guesses {
+			seedCustomerProfile(f, t, name, fixtureNodeRoleName, ownership)
+		}
+		createReadyNodegroup(t, f, "c1", "ng1", 1)
+		rec, _ := ownedProfile(t, f, "c1", "ng1")
+		require.NotContains(t, guesses, rec.InstanceProfileName, "the name must not be guessable")
+
+		_, err := deleteNodegroupCall(f, "c1", "ng1")
+		require.NoError(t, err)
+		sweepNodegroups(t, f)
+
+		m := fixtureIAM(t, f)
+		for _, name := range guesses {
+			require.Contains(t, m.Profiles, name, "customer profile %q was deleted", name)
+			assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[name]), "customer profile %q lost its role", name)
+		}
+	})
+
+	t.Run("recorded profile was swapped for one holding another role", func(t *testing.T) {
+		t.Parallel()
+		f := newEKSServiceFixture(t)
+		seedActiveClusterWithToken(t, f, "c1")
+		createReadyNodegroup(t, f, "c1", "ng1", 1)
+		rec, p := ownedProfile(t, f, "c1", "ng1")
+		p.Roles = []*iam.Role{{RoleName: aws.String("customer-role")}}
+
+		_, err := deleteNodegroupCall(f, "c1", "ng1")
+		require.NoError(t, err)
+		sweepNodegroups(t, f)
+
+		m := fixtureIAM(t, f)
+		require.Contains(t, m.Profiles, rec.InstanceProfileName)
+		assert.Equal(t, []string{"customer-role"}, profileRoleNames(m.Profiles[rec.InstanceProfileName]))
+		assert.Zero(t, m.RemoveRoleCalls)
+		assert.Zero(t, m.DeleteInstanceProfileCalls)
+	})
+
+	t.Run("another nodegroup record names the same profile", func(t *testing.T) {
+		t.Parallel()
+		f := newEKSServiceFixture(t)
+		seedActiveClusterWithToken(t, f, "c1")
+		createReadyNodegroup(t, f, "c1", "ng1", 1)
+		rec, _ := ownedProfile(t, f, "c1", "ng1")
+		require.NoError(t, PutNodegroupRecord(t.Context(), f.kv, &NodegroupRecord{
+			ClusterName: "c1", Name: "ng2", Status: eks.NodegroupStatusActive, NodeRole: fixtureNodeRoleARN,
+			InstanceProfileName: rec.InstanceProfileName, InstanceProfileID: rec.InstanceProfileID,
+		}))
+
+		_, err := deleteNodegroupCall(f, "c1", "ng1")
+		require.NoError(t, err)
+		sweepNodegroups(t, f)
+
+		m := fixtureIAM(t, f)
+		require.Contains(t, m.Profiles, rec.InstanceProfileName)
+		assert.Equal(t, []string{fixtureNodeRoleName}, profileRoleNames(m.Profiles[rec.InstanceProfileName]))
+		assert.Zero(t, m.DeleteInstanceProfileCalls)
+	})
+}
+
+func TestDeleteNodegroup_ProfileAndRecordKeptUntilWorkersProvenGone(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	f.worker.stillLive = []string{rec.InstanceIDs[0]}
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	m := fixtureIAM(t, f)
+	assert.Contains(t, m.Profiles, rec.InstanceProfileName)
+	assert.Zero(t, m.RemoveRoleCalls)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+	got, err := describeNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleting, aws.StringValue(got.Nodegroup.Status))
+	cur, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, rec.InstanceIDs, cur.InstanceIDs, "worker ids stay recorded until they are proven gone")
+	assert.Zero(t, cur.DeleteReapAttempts, "workers still ending is not a failed pass")
+
+	f.worker.mu.Lock()
+	f.worker.stillLive = nil
+	f.worker.mu.Unlock()
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, m.Profiles, rec.InstanceProfileName)
+}
+
+func TestDeleteNodegroup_UndecidableWorkerStateKeepsProfile(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	f.worker.liveErr = errors.New("describe fan-out did not complete")
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+
+	m := fixtureIAM(t, f)
+	assert.Contains(t, m.Profiles, rec.InstanceProfileName)
+	assert.Zero(t, m.DeleteInstanceProfileCalls)
+}
+
+func TestDeleteNodegroup_KickFinishesTheDeleteWithoutAReaperSweep(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	f.svc.noDeleteKick = false
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	f.svc.WaitNodegroupDeletes()
+
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
+}
+
+func TestDeleteNodegroup_WorkerStuckPastDeadlineEndsDeleteFailedKeepingProfile(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 2)
+	rec, _ := ownedProfile(t, f, "c1", "ng1")
+	stuck := rec.InstanceIDs[0]
+	f.worker.stillLive = []string{stuck}
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+
+	sweepNodegroups(t, f)
+	cur, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.NoError(t, err)
+	require.Equal(t, eks.NodegroupStatusDeleting, cur.Status, "within the deadline the group keeps waiting")
+
+	backdateNodegroupReap(t, f, "c1", "ng1")
+	sweepNodegroups(t, f)
+
+	desc, err := describeNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	assert.Equal(t, eks.NodegroupStatusDeleteFailed, aws.StringValue(desc.Nodegroup.Status))
+	require.NotEmpty(t, desc.Nodegroup.Health.Issues)
+	issue := desc.Nodegroup.Health.Issues[0]
+	assert.Equal(t, eks.NodegroupIssueCodeNodeTerminationFailure, aws.StringValue(issue.Code))
+	assert.Contains(t, aws.StringValue(issue.Message), stuck)
+	assert.Contains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
+
+	f.worker.mu.Lock()
+	f.worker.stillLive = nil
+	f.worker.mu.Unlock()
+	_, err = deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+	_, err = describeNodegroupCall(f, "c1", "ng1")
+	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
+	assert.NotContains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
+}

spinifex/domains/eks/cluster/nodegroup_profile.go

@@ -0,0 +1,174 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+const (
+	nodegroupProfilePrefix = "eks-"
+
+	profileClusterTagKey   = "spinifex:eks-cluster"
+	profileNodegroupTagKey = "spinifex:eks-nodegroup"
+	profileIDTagKey        = "spinifex:eks-profile-id"
+)
+
+// nodeProfileForLaunch returns the ARN of the instance profile a worker of rec
+// launches with. A record that names no profile of its own predates per-group
+// profiles and keeps the role-named one it already uses.
+func (s *EKSServiceImpl) nodeProfileForLaunch(accountID string, rec *NodegroupRecord) (string, error) {
+	if rec.InstanceProfileName == "" {
+		return s.ensureNodeInstanceProfile(accountID, rec.NodeRole)
+	}
+	return s.ensureNodegroupProfile(accountID, rec)
+}
+
+func nodegroupProfileTags(rec *NodegroupRecord) []*iam.Tag {
+	return []*iam.Tag{
+		{Key: aws.String(profileClusterTagKey), Value: aws.String(rec.ClusterName)},
+		{Key: aws.String(profileNodegroupTagKey), Value: aws.String(rec.Name)},
+		{Key: aws.String(profileIDTagKey), Value: aws.String(rec.InstanceProfileID)},
+	}
+}
+
+// profileOwnedBy reports whether p carries the ownership tags of this exact
+// incarnation of rec.
+func profileOwnedBy(p *iam.InstanceProfile, rec *NodegroupRecord) bool {
+	have := map[string]string{}
+	for _, t := range p.Tags {
+		have[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
+	}
+	for _, want := range nodegroupProfileTags(rec) {
+		if have[aws.StringValue(want.Key)] != aws.StringValue(want.Value) {
+			return false
+		}
+	}
+	return true
+}
+
+func roleNameFromARN(roleARN string) string {
+	return roleARN[strings.LastIndex(roleARN, "/")+1:]
+}
+
+// ensureNodegroupProfile finds or creates the instance profile recorded for
+// rec, tagged as its own, and makes sure it holds the node role. Concurrent
+// worker launches converge on the one profile.
+func (s *EKSServiceImpl) ensureNodegroupProfile(accountID string, rec *NodegroupRecord) (string, error) {
+	roleName, err := s.resolveNodeRoleName(accountID, rec.NodeRole)
+	if err != nil {
+		return "", err
+	}
+	name := rec.InstanceProfileName
+	ens := s.iamEnsurer()
+
+	got, err := ens.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String(name)})
+	if err != nil {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
+			return "", fmt.Errorf("get instance profile %q: %w", name, err)
+		}
+		_, cerr := ens.CreateInstanceProfile(accountID, &iam.CreateInstanceProfileInput{
+			InstanceProfileName: aws.String(name),
+			Tags:                nodegroupProfileTags(rec),
+		})
+		if cerr != nil {
+			// A racing launch may have created it first; re-read and converge.
+			if code, _ := awserrors.ResolveErrorCode(cerr); code != awserrors.ErrorIAMEntityAlreadyExists {
+				return "", fmt.Errorf("create instance profile %q: %w", name, cerr)
+			}
+		}
+		got, err = ens.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String(name)})
+		if err != nil {
+			return "", fmt.Errorf("re-get instance profile %q: %w", name, err)
+		}
+	}
+	if !profileOwnedBy(got.InstanceProfile, rec) {
+		return "", fmt.Errorf("instance profile %q exists but is not this nodegroup's", name)
+	}
+	return s.attachRoleToProfile(accountID, name, roleName,
+		aws.StringValue(got.InstanceProfile.Arn), len(got.InstanceProfile.Roles) > 0)
+}
+
+// deleteNodegroupProfile removes the profile recorded for rec, and only that
+// one: it must be recorded, tagged for this incarnation, hold exactly the node
+// role and be named by no other record. Anything else is left alone.
+func (s *EKSServiceImpl) deleteNodegroupProfile(ctx context.Context, accountID string, kv jetstream.KeyValue, rec *NodegroupRecord) error {
+	name := rec.InstanceProfileName
+	ens := s.iamEnsurer()
+	if name == "" || ens == nil {
+		return nil
+	}
+	got, err := ens.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String(name)})
+	if err != nil {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
+			return nil
+		}
+		return fmt.Errorf("get instance profile %q: %w", name, err)
+	}
+	p := got.InstanceProfile
+
+	if !profileOwnedBy(p, rec) {
+		slog.WarnContext(ctx, "eks: leaving instance profile whose ownership tags do not match its nodegroup",
+			"cluster", rec.ClusterName, "nodegroup", rec.Name, "profile", name)
+		return nil
+	}
+	nodeRole := roleNameFromARN(rec.NodeRole)
+	if len(p.Roles) > 1 || (len(p.Roles) == 1 && aws.StringValue(p.Roles[0].RoleName) != nodeRole) {
+		slog.WarnContext(ctx, "eks: leaving instance profile that holds a role other than the node role",
+			"cluster", rec.ClusterName, "nodegroup", rec.Name, "profile", name)
+		return nil
+	}
+	shared, err := s.profileNamedByOtherRecord(ctx, kv, rec)
+	if err != nil {
+		return err
+	}
+	if shared {
+		slog.WarnContext(ctx, "eks: leaving instance profile another nodegroup record names",
+			"cluster", rec.ClusterName, "nodegroup", rec.Name, "profile", name)
+		return nil
+	}
+
+	if len(p.Roles) == 1 {
+		_, err := ens.RemoveRoleFromInstanceProfile(accountID, &iam.RemoveRoleFromInstanceProfileInput{
+			InstanceProfileName: aws.String(name), RoleName: aws.String(nodeRole),
+		})
+		if err != nil {
+			if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
+				return fmt.Errorf("remove role %q from instance profile %q: %w", nodeRole, name, err)
+			}
+		}
+	}
+	if _, err := ens.DeleteInstanceProfile(accountID, &iam.DeleteInstanceProfileInput{InstanceProfileName: aws.String(name)}); err != nil {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
+			return fmt.Errorf("delete instance profile %q: %w", name, err)
+		}
+	}
+	return nil
+}
+
+// profileNamedByOtherRecord reports whether any nodegroup record in the
+// account other than rec names rec's profile.
+func (s *EKSServiceImpl) profileNamedByOtherRecord(ctx context.Context, kv jetstream.KeyValue, rec *NodegroupRecord) (bool, error) {
+	clusters, err := listClusterNames(ctx, kv)
+	if err != nil {
+		return false, fmt.Errorf("list clusters: %w", err)
+	}
+	for _, cluster := range clusters {
+		recs, err := ListNodegroupRecords(ctx, kv, cluster)
+		if err != nil {
+			return false, fmt.Errorf("list nodegroups of %s: %w", cluster, err)
+		}
+		for _, other := range recs {
+			if other.InstanceProfileName == rec.InstanceProfileName && (other.ClusterName != rec.ClusterName || other.Name != rec.Name) {
+				return true, nil
+			}
+		}
+	}
+	return false, nil
+}

spinifex/domains/eks/cluster/nodegroup_test.go

@@ -86,6 +86,12 @@ type fakeWorkerLauncher struct {
 	// before subsequent calls succeed — simulates a transient launch failure
 	// (QMP timeout, host pressure) that clears on retry.
 	failNextN int
+	// beforeRun, when set, runs at the start of every launch, before it is
+	// recorded, so a test can inspect durable state at launch time.
+	beforeRun func(*ec2.RunInstancesInput)
+	// stillLive lists instances LiveWorkerInstances keeps reporting as live.
+	stillLive []string
+	liveErr   error
 }
 
 var _ WorkerLauncher = (*fakeWorkerLauncher)(nil)
@@ -101,6 +107,9 @@ func (f *fakeWorkerLauncher) RunWorkerInstance(ctx context.Context, input *ec2.R
 func (f *fakeWorkerLauncher) RunWorkerInstanceOnNode(_ context.Context, nodeID string, input *ec2.RunInstancesInput, _ string) (*ec2.Reservation, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
+	if f.beforeRun != nil {
+		f.beforeRun(input)
+	}
 	f.runCalls = append(f.runCalls, input)
 	f.runNodes = append(f.runNodes, nodeID)
 	if f.failNextN > 0 {
@@ -129,6 +138,21 @@ func (f *fakeWorkerLauncher) TerminateWorkerInstances(_ context.Context, ids []s
 	return f.termErr
 }
 
+func (f *fakeWorkerLauncher) LiveWorkerInstances(_ context.Context, ids []string, _ string) ([]string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.liveErr != nil {
+		return nil, f.liveErr
+	}
+	var live []string
+	for _, id := range ids {
+		if slices.Contains(f.stillLive, id) {
+			live = append(live, id)
+		}
+	}
+	return live, nil
+}
+
 // runCount / terminatedCount return the recorded call totals under the lock so
 // concurrent tests can read them safely after joining.
 func (f *fakeWorkerLauncher) runCount() int {
@@ -927,7 +951,7 @@ func TestUpdateNodegroupConfig_CapacityErrorSurfacesCode(t *testing.T) {
 	})
 }
 
-func TestDeleteNodegroup_TerminatesAndIsIdempotent(t *testing.T) {
+func TestDeleteNodegroup_ReapedGroupIsGoneAndRepeatDeleteIsNotFound(t *testing.T) {
 	t.Parallel()
 	f := newEKSServiceFixture(t)
 	seedActiveClusterWithToken(t, f, "c1")
@@ -940,13 +964,16 @@ func TestDeleteNodegroup_TerminatesAndIsIdempotent(t *testing.T) {
 		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1"),
 	}, testAccountID)
 	require.NoError(t, err)
+	_, err = f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
+	require.NoError(t, err)
+	f.svc.WaitNodegroupDeletes()
 	require.Len(t, f.worker.terminateCalls, 1)
 	assert.Len(t, f.worker.terminateCalls[0], 2)
 
 	_, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
 	assert.ErrorIs(t, err, ErrNodegroupNotFound)
 
-	// Second delete → ResourceNotFoundException (record already gone).
+	// Once the reaper has finished the group is gone: ResourceNotFoundException.
 	_, err = f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
 		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng1"),
 	}, testAccountID)

spinifex/domains/eks/cluster/service_impl.go

@@ -201,20 +201,26 @@ type WorkerLauncher interface {
 	// host spread. An empty nodeID launches on the local node like RunWorkerInstance.
 	RunWorkerInstanceOnNode(ctx context.Context, nodeID string, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error)
 	TerminateWorkerInstances(ctx context.Context, instanceIDs []string, accountID string) error
+	// LiveWorkerInstances returns the given workers not yet proven terminated or
+	// absent. An undecidable answer is an error, never an empty list.
+	LiveWorkerInstances(ctx context.Context, instanceIDs []string, accountID string) ([]string, error)
 }
 
 // iamEnsurer resolves the IAM ensurer, preferring the test-injected deps.IAM,
 // then the lazy IAMProvider (built at launch time to dodge the daemon-startup
 // NATS-KV race). Returns nil when neither yields a service, so callers fall back
-// to static creds / skip the profile. The provider's concrete service satisfies
-// instanceProfileEnsurer (identical method set to SystemInstanceRoleEnsurer).
+// to static creds / skip the profile.
 func (s *EKSServiceImpl) iamEnsurer() instanceProfileEnsurer {
 	if s.deps.IAM != nil {
 		return s.deps.IAM
 	}
 	if s.deps.IAMProvider != nil {
 		if e := s.deps.IAMProvider(); e != nil {
-			return e
+			// The concrete service has the wider surface; the shared interface
+			// stays narrow so other callers are not forced to grow it.
+			if ep, ok := e.(instanceProfileEnsurer); ok {
+				return ep
+			}
 		}
 	}
 	return nil
@@ -227,6 +233,8 @@ type instanceProfileEnsurer interface {
 	GetInstanceProfile(accountID string, input *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error)
 	CreateInstanceProfile(accountID string, input *iam.CreateInstanceProfileInput) (*iam.CreateInstanceProfileOutput, error)
 	AddRoleToInstanceProfile(accountID string, input *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error)
+	RemoveRoleFromInstanceProfile(accountID string, input *iam.RemoveRoleFromInstanceProfileInput) (*iam.RemoveRoleFromInstanceProfileOutput, error)
+	DeleteInstanceProfile(accountID string, input *iam.DeleteInstanceProfileInput) (*iam.DeleteInstanceProfileOutput, error)
 
 	// Role find-or-create surface for the system-managed control-plane role.
 	// Unlike worker node roles (customer-supplied), the k3s server role is
@@ -276,6 +284,12 @@ type EKSServiceImpl struct {
 	teardownMu      sync.Mutex
 	teardownRunning map[string]struct{}
 	teardownWG      sync.WaitGroup
+	// noDeleteKick, set by tests, leaves a nodegroup delete to the reaper sweep.
+	noDeleteKick bool
+	// workersEndWait and workersEndPoll bound how long one delete pass waits for
+	// ended workers to be proven gone; zero means the defaults.
+	workersEndWait time.Duration
+	workersEndPoll time.Duration
 	// teardownKV, when set by a test, replaces the converger's account bucket.
 	teardownKV func(ctx context.Context, accountID string) (jetstream.KeyValue, error)
 

spinifex/domains/eks/cluster/types.go

@@ -49,6 +49,16 @@ type NodegroupRecord struct {
 	Tags           map[string]string `json:"tags,omitempty"`
 	InstanceIDs    []string          `json:"instanceIds,omitempty"`
 	Health         string            `json:"health,omitempty"`
+	// InstanceProfileID and InstanceProfileName identify the instance profile
+	// EKS made for this incarnation of the nodegroup; both are empty on a record
+	// that predates it.
+	InstanceProfileID   string `json:"instanceProfileId,omitempty"`
+	InstanceProfileName string `json:"instanceProfileName,omitempty"`
+	// DeletingSince, DeleteReapAttempts and LastDeleteReapAttempt pace the reaper that finishes
+	// a DELETING nodegroup.
+	DeletingSince         time.Time `json:"deletingSince,omitzero"`
+	DeleteReapAttempts    int       `json:"deleteReapAttempts,omitempty"`
+	LastDeleteReapAttempt time.Time `json:"lastDeleteReapAttempt,omitzero"`
 	// GPUEnabled/GPUVendor cache whether InstanceTypes includes a GPU family and
 	// which vendor, so worker launches resolve the matching GPU node AMI without
 	// re-scanning InstanceTypes on every launch.

spinifex/domains/eks/cluster/worker_membership_test.go

@@ -56,6 +56,9 @@ func TestListManagedWorkers_DropsWorkersTheNodegroupTerminated(t *testing.T) {
 		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng2"),
 	}, testAccountID)
 	require.NoError(t, err)
+	_, err = f.svc.NewNodegroupDeletingReaper().Sweep(context.Background())
+	require.NoError(t, err)
+	f.svc.WaitNodegroupDeletes()
 	got = managedWorkers(t, f, "c1")
 	assert.Len(t, got, 1)
 	for _, id := range ng2.InstanceIDs {

spinifex/domains/iam/mock/mock.go

@@ -36,6 +36,9 @@ type SystemInstanceRoleEnsurer struct {
 	CreateRoleCalls               int
 	CreateInstanceProfileCalls    int
 	AddRoleToInstanceProfileCalls int
+	TagInstanceProfileCalls       int
+	RemoveRoleCalls               int
+	DeleteInstanceProfileCalls    int
 
 	// LastTrustDoc captures the AssumeRolePolicyDocument of the most recent
 	// CreateRole call.
@@ -54,6 +57,14 @@ type SystemInstanceRoleEnsurer struct {
 	// AddRoleToInstanceProfileErr, when set, is returned by
 	// AddRoleToInstanceProfile instead of attaching.
 	AddRoleToInstanceProfileErr error
+	// RemoveRoleFromInstanceProfileErr and DeleteInstanceProfileErr, when set,
+	// are returned by the matching call instead of acting.
+	RemoveRoleFromInstanceProfileErr error
+	DeleteInstanceProfileErr         error
+	// GetInstanceProfileErr, when set, is returned by GetInstanceProfile
+	// instead of reading, for exercising callers that must tell a failed read
+	// from a missing profile.
+	GetInstanceProfileErr error
 	// EmptyInstanceProfileARN makes CreateInstanceProfile return an empty
 	// ARN, for exercising callers that must reject it.
 	EmptyInstanceProfileARN bool
@@ -81,6 +92,7 @@ func copyRole(r *iam.Role) *iam.Role {
 func copyProfile(p *iam.InstanceProfile) *iam.InstanceProfile {
 	c := *p
 	c.Roles = append([]*iam.Role(nil), p.Roles...)
+	c.Tags = append([]*iam.Tag(nil), p.Tags...)
 	return &c
 }
 
@@ -130,6 +142,9 @@ func (e *SystemInstanceRoleEnsurer) GetInstanceProfile(_ string, in *iam.GetInst
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
+	if e.GetInstanceProfileErr != nil {
+		return nil, e.GetInstanceProfileErr
+	}
 	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
 	if !ok {
 		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
@@ -157,6 +172,7 @@ func (e *SystemInstanceRoleEnsurer) CreateInstanceProfile(accountID string, in *
 	p := &iam.InstanceProfile{
 		InstanceProfileName: aws.String(name),
 		Arn:                 aws.String(arn),
+		Tags:                append([]*iam.Tag(nil), in.Tags...),
 	}
 	e.Profiles[name] = p
 	return &iam.CreateInstanceProfileOutput{InstanceProfile: copyProfile(p)}, nil
@@ -181,3 +197,101 @@ func (e *SystemInstanceRoleEnsurer) AddRoleToInstanceProfile(_ string, in *iam.A
 	p.Roles = []*iam.Role{{RoleName: in.RoleName}}
 	return &iam.AddRoleToInstanceProfileOutput{}, nil
 }
+
+func (e *SystemInstanceRoleEnsurer) TagInstanceProfile(_ string, in *iam.TagInstanceProfileInput) (*iam.TagInstanceProfileOutput, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	e.TagInstanceProfileCalls++
+	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
+	if !ok {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
+	}
+	for _, t := range in.Tags {
+		replaced := false
+		for i, have := range p.Tags {
+			if aws.StringValue(have.Key) == aws.StringValue(t.Key) {
+				p.Tags[i] = t
+				replaced = true
+				break
+			}
+		}
+		if !replaced {
+			p.Tags = append(p.Tags, t)
+		}
+	}
+	return &iam.TagInstanceProfileOutput{}, nil
+}
+
+func (e *SystemInstanceRoleEnsurer) UntagInstanceProfile(_ string, in *iam.UntagInstanceProfileInput) (*iam.UntagInstanceProfileOutput, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
+	if !ok {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
+	}
+	drop := map[string]bool{}
+	for _, k := range in.TagKeys {
+		drop[aws.StringValue(k)] = true
+	}
+	kept := p.Tags[:0:0]
+	for _, t := range p.Tags {
+		if !drop[aws.StringValue(t.Key)] {
+			kept = append(kept, t)
+		}
+	}
+	p.Tags = kept
+	return &iam.UntagInstanceProfileOutput{}, nil
+}
+
+func (e *SystemInstanceRoleEnsurer) ListInstanceProfileTags(_ string, in *iam.ListInstanceProfileTagsInput) (*iam.ListInstanceProfileTagsOutput, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
+	if !ok {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
+	}
+	return &iam.ListInstanceProfileTagsOutput{Tags: append([]*iam.Tag(nil), p.Tags...)}, nil
+}
+
+func (e *SystemInstanceRoleEnsurer) RemoveRoleFromInstanceProfile(_ string, in *iam.RemoveRoleFromInstanceProfileInput) (*iam.RemoveRoleFromInstanceProfileOutput, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	e.RemoveRoleCalls++
+	if e.RemoveRoleFromInstanceProfileErr != nil {
+		return nil, e.RemoveRoleFromInstanceProfileErr
+	}
+	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
+	if !ok {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
+	}
+	if len(p.Roles) == 0 || aws.StringValue(p.Roles[0].RoleName) != aws.StringValue(in.RoleName) {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The role with name %s cannot be found.", aws.StringValue(in.RoleName))
+	}
+	p.Roles = nil
+	return &iam.RemoveRoleFromInstanceProfileOutput{}, nil
+}
+
+// DeleteInstanceProfile refuses while a role is attached, as IAM does.
+func (e *SystemInstanceRoleEnsurer) DeleteInstanceProfile(_ string, in *iam.DeleteInstanceProfileInput) (*iam.DeleteInstanceProfileOutput, error) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	e.DeleteInstanceProfileCalls++
+	if e.DeleteInstanceProfileErr != nil {
+		return nil, e.DeleteInstanceProfileErr
+	}
+	name := aws.StringValue(in.InstanceProfileName)
+	p, ok := e.Profiles[name]
+	if !ok {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", name)
+	}
+	if len(p.Roles) > 0 {
+		return nil, awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first.")
+	}
+	delete(e.Profiles, name)
+	return &iam.DeleteInstanceProfileOutput{}, nil
+}

spinifex/domains/iam/mock/mock_test.go

@@ -83,3 +83,112 @@ func TestPutRolePolicy_InjectedErrorStillRecordsCall(t *testing.T) {
 		t.Fatalf("RolePolicies[r1] = %q, want %q", e.RolePolicies["r1"], "doc")
 	}
 }
+
+func seedProfile(t *testing.T, e *mock.SystemInstanceRoleEnsurer, name string) {
+	t.Helper()
+	if _, err := e.CreateInstanceProfile("acct", &iam.CreateInstanceProfileInput{InstanceProfileName: aws.String(name)}); err != nil {
+		t.Fatalf("CreateInstanceProfile: %v", err)
+	}
+}
+
+func requireCode(t *testing.T, err error, want string) {
+	t.Helper()
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != want {
+		t.Fatalf("err = %v, want %s", err, want)
+	}
+}
+
+func TestInstanceProfileTags_TagReplacesSameKeyUntagDropsListReturnsCopy(t *testing.T) {
+	e := mock.New()
+	seedProfile(t, e, "p1")
+	tag := func(k, v string) *iam.Tag { return &iam.Tag{Key: aws.String(k), Value: aws.String(v)} }
+	if _, err := e.TagInstanceProfile("acct", &iam.TagInstanceProfileInput{
+		InstanceProfileName: aws.String("p1"), Tags: []*iam.Tag{tag("a", "1"), tag("b", "2")},
+	}); err != nil {
+		t.Fatalf("TagInstanceProfile: %v", err)
+	}
+	if _, err := e.TagInstanceProfile("acct", &iam.TagInstanceProfileInput{
+		InstanceProfileName: aws.String("p1"), Tags: []*iam.Tag{tag("a", "9")},
+	}); err != nil {
+		t.Fatalf("re-tag: %v", err)
+	}
+	if _, err := e.UntagInstanceProfile("acct", &iam.UntagInstanceProfileInput{
+		InstanceProfileName: aws.String("p1"), TagKeys: aws.StringSlice([]string{"b"}),
+	}); err != nil {
+		t.Fatalf("UntagInstanceProfile: %v", err)
+	}
+	out, err := e.ListInstanceProfileTags("acct", &iam.ListInstanceProfileTagsInput{InstanceProfileName: aws.String("p1")})
+	if err != nil {
+		t.Fatalf("ListInstanceProfileTags: %v", err)
+	}
+	if len(out.Tags) != 1 || aws.StringValue(out.Tags[0].Key) != "a" || aws.StringValue(out.Tags[0].Value) != "9" {
+		t.Fatalf("tags = %v, want only a=9", out.Tags)
+	}
+}
+
+func TestInstanceProfileTags_MissingProfileIsNoSuchEntity(t *testing.T) {
+	e := mock.New()
+	name := aws.String("absent")
+	_, err := e.TagInstanceProfile("acct", &iam.TagInstanceProfileInput{InstanceProfileName: name})
+	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
+	_, err = e.UntagInstanceProfile("acct", &iam.UntagInstanceProfileInput{InstanceProfileName: name})
+	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
+	_, err = e.ListInstanceProfileTags("acct", &iam.ListInstanceProfileTagsInput{InstanceProfileName: name})
+	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
+}
+
+func TestDeleteInstanceProfile_RefusesWhileRoleAttachedThenDeletes(t *testing.T) {
+	e := mock.New()
+	seedProfile(t, e, "p1")
+	if _, err := e.AddRoleToInstanceProfile("acct", &iam.AddRoleToInstanceProfileInput{
+		InstanceProfileName: aws.String("p1"), RoleName: aws.String("r1"),
+	}); err != nil {
+		t.Fatalf("AddRoleToInstanceProfile: %v", err)
+	}
+	del := &iam.DeleteInstanceProfileInput{InstanceProfileName: aws.String("p1")}
+	_, err := e.DeleteInstanceProfile("acct", del)
+	requireCode(t, err, awserrors.ErrorIAMDeleteConflict)
+	if _, ok := e.Profiles["p1"]; !ok {
+		t.Fatal("a refused delete must leave the profile in place")
+	}
+
+	_, err = e.RemoveRoleFromInstanceProfile("acct", &iam.RemoveRoleFromInstanceProfileInput{
+		InstanceProfileName: aws.String("p1"), RoleName: aws.String("other"),
+	})
+	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
+	if _, err := e.RemoveRoleFromInstanceProfile("acct", &iam.RemoveRoleFromInstanceProfileInput{
+		InstanceProfileName: aws.String("p1"), RoleName: aws.String("r1"),
+	}); err != nil {
+		t.Fatalf("RemoveRoleFromInstanceProfile: %v", err)
+	}
+	if _, err := e.DeleteInstanceProfile("acct", del); err != nil {
+		t.Fatalf("DeleteInstanceProfile: %v", err)
+	}
+	if _, ok := e.Profiles["p1"]; ok {
+		t.Fatal("profile still present after delete")
+	}
+	_, err = e.DeleteInstanceProfile("acct", del)
+	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
+}
+
+func TestInstanceProfileMutations_MissingProfileAndInjectedErrors(t *testing.T) {
+	e := mock.New()
+	name := aws.String("absent")
+	_, err := e.RemoveRoleFromInstanceProfile("acct", &iam.RemoveRoleFromInstanceProfileInput{InstanceProfileName: name, RoleName: aws.String("r")})
+	requireCode(t, err, awserrors.ErrorIAMNoSuchEntity)
+
+	boom := errors.New("boom")
+	e.RemoveRoleFromInstanceProfileErr, e.DeleteInstanceProfileErr, e.GetInstanceProfileErr = boom, boom, boom
+	if _, err := e.RemoveRoleFromInstanceProfile("acct", &iam.RemoveRoleFromInstanceProfileInput{InstanceProfileName: name, RoleName: aws.String("r")}); !errors.Is(err, boom) {
+		t.Fatalf("RemoveRole err = %v, want injected", err)
+	}
+	if _, err := e.DeleteInstanceProfile("acct", &iam.DeleteInstanceProfileInput{InstanceProfileName: name}); !errors.Is(err, boom) {
+		t.Fatalf("Delete err = %v, want injected", err)
+	}
+	if _, err := e.GetInstanceProfile("acct", &iam.GetInstanceProfileInput{InstanceProfileName: name}); !errors.Is(err, boom) {
+		t.Fatalf("Get err = %v, want injected", err)
+	}
+	if e.RemoveRoleCalls != 2 || e.DeleteInstanceProfileCalls != 1 {
+		t.Fatalf("calls = remove %d delete %d, want 2 and 1", e.RemoveRoleCalls, e.DeleteInstanceProfileCalls)
+	}
+}

spinifex/domains/lambda/awsapi/get.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"log/slog"
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -18,47 +19,109 @@ import (
 // codeURLTTL is how long the presigned code location stays valid.
 const codeURLTTL = 10 * time.Minute
 
+// maxListItems is the largest MaxItems a list action accepts.
+const maxListItems = 10000
+
 func notFound(region, accountID, name string) error {
 	return awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "Function not found: %s",
 		arn.FormatLambdaFunction(region, accountID, name))
 }
 
-// getFunction implements GetFunction for the $LATEST version.
-func (d Deps) getFunction(ctx context.Context, c *call) (reply, error) {
+// lookup resolves the requested $LATEST function's record; query is the
+// Qualifier query parameter, empty for an action that has none.
+func (d Deps) lookup(ctx context.Context, c *call, query string) (function.Record, uint64, error) {
 	name, q, err := resolveFunctionName(c.Region, c.AccountID, c.Params[0])
 	if err != nil {
-		return reply{}, err
+		return function.Record{}, 0, err
 	}
-	if query := c.Query.Get("Qualifier"); query != "" {
+	if query != "" {
 		if q != "" && q != query {
-			return reply{}, invalid("The derived qualifier from the function name does not match the specified qualifier.")
+			return function.Record{}, 0, invalid("The derived qualifier from the function name does not match the specified qualifier.")
 		}
 		q = query
 	}
 	if q != "" && q != "$LATEST" {
-		return reply{}, notFound(c.Region, c.AccountID, name+":"+q)
+		return function.Record{}, 0, notFound(c.Region, c.AccountID, name+":"+q)
 	}
 	rec, revision, err := d.Meta.Get(ctx, c.AccountID, c.Region, name)
 	if errors.Is(err, function.ErrNotFound) {
-		return reply{}, notFound(c.Region, c.AccountID, name)
+		return function.Record{}, 0, notFound(c.Region, c.AccountID, name)
 	}
+	return rec, revision, err
+}
+
+// loadConfiguration resolves the requested $LATEST function and renders its
+// configuration; it is a metadata read and never touches the code store.
+func (d Deps) loadConfiguration(ctx context.Context, c *call, query string) (function.Record, *lambda.FunctionConfiguration, error) {
+	rec, revision, err := d.lookup(ctx, c, query)
 	if err != nil {
-		return reply{}, err
+		return function.Record{}, nil, err
 	}
 
 	var env map[string]string
 	if rec.EnvCipher != "" {
 		if d.AEAD == nil {
-			return reply{}, serviceError("environment decryption is unavailable")
+			return function.Record{}, nil, serviceError("environment decryption is unavailable")
 		}
 		if env, err = function.OpenEnv(d.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher); err != nil {
 			slog.ErrorContext(ctx, "Lambda: open environment failed", "function_id", rec.FunctionID, "err", err)
-			return reply{}, serviceError("could not read the function environment")
+			return function.Record{}, nil, serviceError("could not read the function environment")
 		}
 	}
 	cfg, err := configuration(rec, revision, env)
 	if err != nil {
-		return reply{}, serviceError("could not build the function configuration")
+		return function.Record{}, nil, serviceError("could not build the function configuration")
+	}
+	return rec, cfg, nil
+}
+
+// getFunctionConfiguration implements GetFunctionConfiguration for $LATEST.
+func (d Deps) getFunctionConfiguration(ctx context.Context, c *call) (reply, error) {
+	_, cfg, err := d.loadConfiguration(ctx, c, c.Query.Get("Qualifier"))
+	if err != nil {
+		return reply{}, err
+	}
+	return reply{Status: http.StatusOK, Body: cfg}, nil
+}
+
+// noCodeSigning is AWS's body for a function without a code signing config:
+// both members present and null, which the SDK shapes cannot encode.
+var noCodeSigning = rawJSON(`{"CodeSigningConfigArn":null,"FunctionName":null}`)
+
+// getFunctionCodeSigningConfig implements GetFunctionCodeSigningConfig; code
+// signing is unsupported, so it only proves the function exists.
+func (d Deps) getFunctionCodeSigningConfig(ctx context.Context, c *call) (reply, error) {
+	if _, _, err := d.lookup(ctx, c, ""); err != nil {
+		return reply{}, err
+	}
+	return reply{Status: http.StatusOK, Body: noCodeSigning}, nil
+}
+
+// listVersionsByFunction implements ListVersionsByFunction while $LATEST is
+// the only version. Paging is refused rather than guessed at.
+func (d Deps) listVersionsByFunction(ctx context.Context, c *call) (reply, error) {
+	if raw, ok := c.Query["MaxItems"]; ok {
+		n, err := strconv.Atoi(raw[0])
+		if err != nil || n < 1 || n > maxListItems {
+			return reply{}, invalid("MaxItems must be between 1 and %d", maxListItems)
+		}
+	}
+	if c.Query.Has("Marker") {
+		return reply{}, unsupported("Marker")
+	}
+	rec, cfg, err := d.loadConfiguration(ctx, c, "")
+	if err != nil {
+		return reply{}, err
+	}
+	cfg.FunctionArn = aws.String(arn.FormatLambdaFunction(rec.Region, rec.AccountID, rec.FunctionName) + ":$LATEST")
+	return reply{Status: http.StatusOK, Body: &lambda.ListVersionsByFunctionOutput{Versions: []*lambda.FunctionConfiguration{cfg}}}, nil
+}
+
+// getFunction implements GetFunction for the $LATEST version.
+func (d Deps) getFunction(ctx context.Context, c *call) (reply, error) {
+	rec, cfg, err := d.loadConfiguration(ctx, c, c.Query.Get("Qualifier"))
+	if err != nil {
+		return reply{}, err
 	}
 	out := &lambda.GetFunctionOutput{
 		Configuration: cfg,

spinifex/domains/lambda/awsapi/handler_test.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"runtime"
 	"strings"
 	"sync"
@@ -662,8 +663,316 @@ func TestAuthorization(t *testing.T) {
 
 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateFunction", "GetFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "ListVersionsByFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
+	assert.Equal(t, "GetFunctionCodeSigningConfig", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2020-06-30/functions/x/code-signing-config", nil)))
 	assert.Equal(t, "CreateFunction", reg.ResolveAction(httptest.NewRequest(http.MethodPost, "/2015-03-31/functions", nil)))
 	assert.Equal(t, "GetFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x", nil)))
+	assert.Equal(t, "GetFunctionConfiguration", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/configuration", nil)))
 	assert.Empty(t, reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x", nil)))
 }
+
+func TestGetFunctionConfiguration_ReturnsTheSameConfigurationAsGetFunction(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+
+	rec, err := e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+	require.NoError(t, err)
+	var full lambda.GetFunctionOutput
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &full))
+
+	rec, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, rec.Code)
+	assert.NotContains(t, rec.Body.String(), `"Code"`, "the configuration is the top-level body, not GetFunction's wrapper")
+	var cfg lambda.FunctionConfiguration
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &cfg))
+	assert.Equal(t, *full.Configuration, cfg)
+	assert.Equal(t, "prod", *cfg.Environment.Variables["MODE"])
+
+	arn := "arn%3Aaws%3Alambda%3A" + gwtest.Region + "%3A" + gwtest.AccountID + "%3Afunction%3Ahello"
+	for _, path := range []string{
+		"/2015-03-31/functions/hello/configuration?Qualifier=%24LATEST",
+		"/2015-03-31/functions/" + arn + "/configuration",
+		"/2015-03-31/functions/" + arn + "%3A%24LATEST/configuration",
+	} {
+		rec, err := e.do(http.MethodGet, path, nil)
+		require.NoError(t, err, path)
+		assert.Equal(t, http.StatusOK, rec.Code, path)
+	}
+}
+
+func TestGetFunctionConfiguration_MissingAndUnsupportedMatchGetFunction(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	for label, tc := range map[string]struct{ suffix, code string }{
+		"missing":         {"ghost", awserrors.ErrorResourceNotFoundException},
+		"version":         {"hello?Qualifier=3", awserrors.ErrorResourceNotFoundException},
+		"other account":   {"arn%3Aaws%3Alambda%3A" + gwtest.Region + "%3A999999999999%3Afunction%3Ahello", awserrors.ErrorResourceNotFoundException},
+		"qualifier clash": {"hello%3A%24LATEST?Qualifier=3", awserrors.ErrorInvalidParameterValueException},
+		"not lambda":      {"arn%3Aaws%3As3%3A%3A%3Abucket", awserrors.ErrorInvalidParameterValueException},
+	} {
+		t.Run(label, func(t *testing.T) {
+			path, query, _ := strings.Cut(tc.suffix, "?")
+			if query != "" {
+				query = "?" + query
+			}
+			_, getErr := e.do(http.MethodGet, "/2015-03-31/functions/"+path+query, nil)
+			requireCode(t, getErr, tc.code)
+			_, err := e.do(http.MethodGet, "/2015-03-31/functions/"+path+"/configuration"+query, nil)
+			requireCode(t, err, tc.code)
+			assert.Equal(t, getErr.Error(), err.Error())
+		})
+	}
+
+	e.h.Region = "us-west-2"
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestGetFunctionConfiguration_DoesNotNeedTheCodeStore(t *testing.T) {
+	for name, codes := range map[string]awsapi.CodeStoreFunc{
+		"store outage": func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") },
+		"no store":     nil,
+	} {
+		t.Run(name, func(t *testing.T) {
+			e := newEnv(t)
+			_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+			require.NoError(t, err)
+			e.deps.Codes = codes
+
+			rec, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+			require.NoError(t, err)
+			require.Equal(t, http.StatusOK, rec.Code)
+			var cfg lambda.FunctionConfiguration
+			require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &cfg))
+			assert.Equal(t, "hello", *cfg.FunctionName)
+
+			_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+			requireCode(t, err, awserrors.ErrorServiceException)
+		})
+	}
+}
+
+func TestGetFunctionConfiguration_Authorization(t *testing.T) {
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	setup := func(t *testing.T, statements ...iamdomain.Statement) *env {
+		e := newEnv(t, statements...)
+		_, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+			FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+			Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+		})
+		require.NoError(t, err)
+		return e
+	}
+	get := func(e *env) error {
+		_, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+		return err
+	}
+
+	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))))
+	})
+	t.Run("a deny on the function ARN applies", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Deny", "lambda:GetFunctionConfiguration", arn))))
+	})
+	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionConfiguration", arn+"-other"))))
+	})
+	t.Run("an allow on the function ARN passes", func(t *testing.T) {
+		require.NoError(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionConfiguration", arn))))
+	})
+}
+
+func TestGetFunctionCodeSigningConfig_ExistingFunctionHasExplicitNulls(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	arn := "arn%3Aaws%3Alambda%3A" + gwtest.Region + "%3A" + gwtest.AccountID + "%3Afunction%3Ahello"
+	for _, name := range []string{"hello", arn, gwtest.AccountID + "%3Afunction%3Ahello"} {
+		rec, err := e.do(http.MethodGet, "/2020-06-30/functions/"+name+"/code-signing-config", nil)
+		require.NoError(t, err, name)
+		require.Equal(t, http.StatusOK, rec.Code, name)
+		assert.Equal(t, "application/json", rec.Header().Get("Content-Type"))
+		assert.JSONEq(t, `{"CodeSigningConfigArn":null,"FunctionName":null}`, rec.Body.String(), name)
+	}
+}
+
+func TestGetFunctionCodeSigningConfig_MissingFunctionIsNotFound(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	_, err = e.do(http.MethodGet, "/2020-06-30/functions/ghost/code-signing-config", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, getErr := e.do(http.MethodGet, "/2015-03-31/functions/ghost", nil)
+	assert.Equal(t, getErr.Error(), err.Error())
+
+	e.h.Region = "us-west-2"
+	_, err = e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestGetFunctionCodeSigningConfig_NeedsNeitherCodeStoreNorDecryption(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+	e.deps.Codes = func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") }
+	e.deps.AEAD = nil
+
+	rec, err := e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{"CodeSigningConfigArn":null,"FunctionName":null}`, rec.Body.String())
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	requireCode(t, err, awserrors.ErrorServiceException)
+
+	e.deps.Codes = nil
+	_, err = e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+	require.NoError(t, err)
+}
+
+func TestGetFunctionCodeSigningConfig_Authorization(t *testing.T) {
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	setup := func(t *testing.T, statements ...iamdomain.Statement) *env {
+		e := newEnv(t, statements...)
+		_, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+			FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+			Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+		})
+		require.NoError(t, err)
+		return e
+	}
+	get := func(e *env) error {
+		_, err := e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+		return err
+	}
+
+	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))))
+	})
+	t.Run("a deny on the function ARN applies", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Deny", "lambda:GetFunctionCodeSigningConfig", arn))))
+	})
+	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionCodeSigningConfig", arn+"-other"))))
+	})
+	t.Run("an allow on the function ARN passes", func(t *testing.T) {
+		require.NoError(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionCodeSigningConfig", arn))))
+	})
+}
+
+func TestListVersionsByFunction_ListsOnlyLatest(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+	rec, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	require.NoError(t, err)
+	var cfg lambda.FunctionConfiguration
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &cfg))
+
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	for _, path := range []string{
+		"/2015-03-31/functions/hello/versions",
+		"/2015-03-31/functions/hello/versions?MaxItems=10000",
+		"/2015-03-31/functions/hello/versions?MaxItems=1",
+		"/2015-03-31/functions/" + url.PathEscape(arn) + "/versions",
+	} {
+		rec, err := e.do(http.MethodGet, path, nil)
+		require.NoError(t, err, path)
+		require.Equal(t, http.StatusOK, rec.Code, path)
+		assert.NotContains(t, rec.Body.String(), "NextMarker", path)
+		var out lambda.ListVersionsByFunctionOutput
+		require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out), path)
+		require.Len(t, out.Versions, 1, path)
+		v := out.Versions[0]
+		assert.Equal(t, arn+":$LATEST", *v.FunctionArn, path)
+		assert.Equal(t, "$LATEST", *v.Version, path)
+		assert.Equal(t, "prod", *v.Environment.Variables["MODE"], path)
+		v.FunctionArn = cfg.FunctionArn
+		assert.Equal(t, cfg, *v, "apart from the qualified ARN it is the configuration read")
+	}
+}
+
+func TestListVersionsByFunction_RefusesBadMaxItemsAndAnyMarker(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	for _, q := range []string{"MaxItems=0", "MaxItems=10001", "MaxItems=-1", "MaxItems=ten", "MaxItems=", "Marker=not-a-marker", "Marker="} {
+		_, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/versions?"+q, nil)
+		requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+}
+
+func TestListVersionsByFunction_MissingFunctionIsNotFound(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/ghost/versions", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, getErr := e.do(http.MethodGet, "/2015-03-31/functions/ghost", nil)
+	assert.Equal(t, getErr.Error(), err.Error())
+
+	e.h.Region = "us-west-2"
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/versions", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestListVersionsByFunction_DoesNotNeedTheCodeStore(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+	e.deps.Codes = func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") }
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/versions", nil)
+	require.NoError(t, err)
+	e.deps.Codes = nil
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/versions", nil)
+	require.NoError(t, err)
+}
+
+func TestListVersionsByFunction_Authorization(t *testing.T) {
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	setup := func(t *testing.T, statements ...iamdomain.Statement) *env {
+		e := newEnv(t, statements...)
+		_, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+			FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+			Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+		})
+		require.NoError(t, err)
+		return e
+	}
+	list := func(e *env) error {
+		_, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/versions", nil)
+		return err
+	}
+
+	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
+		gwtest.AssertDenied(t, list(setup(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))))
+	})
+	t.Run("a deny on the function ARN applies", func(t *testing.T) {
+		gwtest.AssertDenied(t, list(setup(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Deny", "lambda:ListVersionsByFunction", arn))))
+	})
+	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
+		gwtest.AssertDenied(t, list(setup(t, gwtest.Statement("Allow", "lambda:ListVersionsByFunction", arn+"-other"))))
+	})
+	t.Run("an allow on the function ARN passes", func(t *testing.T) {
+		require.NoError(t, list(setup(t, gwtest.Statement("Allow", "lambda:ListVersionsByFunction", arn))))
+	})
+}

spinifex/domains/lambda/awsapi/registration.go

@@ -117,7 +117,7 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		if decodeErr == nil {
 			resource = functionARN(inv.Region, inv.AccountID, aws.StringValue(c.Create.FunctionName))
 		}
-	case "GetFunction":
+	case "GetFunction", "GetFunctionConfiguration", "GetFunctionCodeSigningConfig", "ListVersionsByFunction":
 		if name, _, err := resolveFunctionName(inv.Region, inv.AccountID, params[0]); err == nil {
 			resource = functionARN(inv.Region, inv.AccountID, name)
 		}
@@ -133,7 +133,10 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 			return err
 		}
 	}
-	if d.Meta == nil || d.Codes == nil {
+	// Only GetFunction presigns code; the other reads are metadata only.
+	needsCodes := action != "GetFunctionConfiguration" && action != "GetFunctionCodeSigningConfig" &&
+		action != "ListVersionsByFunction"
+	if d.Meta == nil || (d.Codes == nil && needsCodes) {
 		return errors.New(awserrors.ErrorServiceException)
 	}
 
@@ -180,7 +183,13 @@ func (d Deps) checkPassRole(inv dispatch.Invocation, roleARN string) error {
 }
 
 func writeJSON(w http.ResponseWriter, out reply) {
-	body, err := jsonutil.BuildJSON(out.Body)
+	var body []byte
+	var err error
+	if raw, ok := out.Body.(rawJSON); ok {
+		body = raw
+	} else {
+		body, err = jsonutil.BuildJSON(out.Body)
+	}
 	if err != nil {
 		slog.Error("Failed to marshal Lambda response", "err", err)
 		w.Header().Set("Content-Type", "application/json")

spinifex/domains/lambda/awsapi/routes.go

@@ -21,6 +21,10 @@ type call struct {
 	Create *lambda.CreateFunctionInput
 }
 
+// rawJSON is a response body written as is, for a shape the SDK encoder
+// cannot produce.
+type rawJSON []byte
+
 // reply is a handler's success: the HTTP status and the SDK shape to encode.
 type reply struct {
 	Status int
@@ -41,6 +45,18 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.getFunction(ctx, c)
 		}},
+	{Method: "GET", Pattern: "/2015-03-31/functions/{FunctionName}/configuration", Action: "GetFunctionConfiguration",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.getFunctionConfiguration(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/functions/{FunctionName}/versions", Action: "ListVersionsByFunction",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.listVersionsByFunction(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2020-06-30/functions/{FunctionName}/code-signing-config", Action: "GetFunctionCodeSigningConfig",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.getFunctionCodeSigningConfig(ctx, c)
+		}},
 }
 
 // actionNames returns the distinct actions in routes in stable order.

spinifex/domains/sqs/storevolume/prepare.go

@@ -31,7 +31,7 @@ type VolumeService interface {
 // Publisher is the provider pair used to reach the new volume's blocks.
 type Publisher interface {
 	PublishVolume(context.Context, ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error)
-	UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) error
+	UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error)
 }
 
 // Request names the volume to create.
@@ -204,9 +204,10 @@ func (p *Preparer) fill(ctx context.Context, volumeID, image string) error {
 func (p *Preparer) unpublish(ctx context.Context, volumeID string) error {
 	cctx, cancel := p.cleanupContext(ctx)
 	defer cancel()
-	return p.Provider.UnpublishVolume(cctx, ebsprovider.UnpublishVolumeRequest{
+	_, err := p.Provider.UnpublishVolume(cctx, ebsprovider.UnpublishVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: p.NodeID,
 	})
+	return err
 }
 
 // fail discards the volume when its export is known closed and otherwise

spinifex/domains/sqs/storevolume/prepare_test.go

@@ -105,14 +105,17 @@ func (h *harness) PublishVolume(_ context.Context, req ebsprovider.PublishVolume
 	return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: export}, nil
 }
 
-func (h *harness) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error {
+func (h *harness) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
 	h.record("unpublish " + req.VolumeID + " on " + req.NodeID)
 	h.unpublishCtxOK = ctx.Err() == nil
 	if h.unpublishBlocks {
 		<-ctx.Done()
-		return ctx.Err()
+		return nil, ctx.Err()
 	}
-	return h.unpublishErr
+	if h.unpublishErr != nil {
+		return nil, h.unpublishErr
+	}
+	return &ebsprovider.UnpublishedVolume{}, nil
 }
 
 func (h *harness) preparer(t *testing.T) *Preparer {

spinifex/operator/accountlifecycle/engine.go

@@ -35,6 +35,12 @@ type Engine struct {
 	// TERMINATING and before the first stage. It records evidence and gates nothing.
 	OnQuiesced func(ctx context.Context, quiescence operatorv1.IAMQuiescence)
 
+	// Cleanup runs the EC2 platform cleanup. Only Exercise uses it.
+	Cleanup PlatformCleaner
+
+	// OnExercised receives what a non-finalizing run did after quiescence.
+	OnExercised func(ctx context.Context, result operatorv1.ExerciseResult)
+
 	// ec2Inventory is EC2's read-only platform inventory. Only DryRun consults
 	// it, and only to log a comparison; no teardown or Inventory call does.
 	ec2Inventory ec2PlatformInventory
@@ -129,7 +135,7 @@ func (e *Engine) Teardown(ctx context.Context, req Request) (*Result, error) {
 		return nil, fmt.Errorf("quiesce account: %w", err)
 	}
 	if e.OnQuiesced != nil {
-		e.OnQuiesced(ctx, e.evaluateQuiescence(ctx, req, generation))
+		e.OnQuiesced(ctx, e.evaluateQuiescence(ctx, req, generation, false))
 	}
 	e.logger().InfoContext(ctx, "Account teardown started",
 		"accountID", account.AccountID, "accountName", account.AccountName, "force", req.Force)
@@ -338,9 +344,9 @@ func SortReapers(reapers []Reaper) {
 
 // evaluateQuiescence reads back what the quiesce wrote and, only if the fence
 // is confirmed, asks EC2 for its platform inventory within a short budget.
-func (e *Engine) evaluateQuiescence(ctx context.Context, req Request, generation uint64) operatorv1.IAMQuiescence {
+func (e *Engine) evaluateQuiescence(ctx context.Context, req Request, generation uint64, exercise bool) operatorv1.IAMQuiescence {
 	ev := QuiesceEvidence{
-		OperationID: req.OperationID, Fence: req.Fence, Generation: generation, EvaluatedAt: time.Now(),
+		OperationID: req.OperationID, Fence: req.Fence, Generation: generation, EvaluatedAt: time.Now(), Exercise: exercise,
 	}
 	ev.Account, ev.ReadbackErr = e.Accounts.GetAccount(req.AccountID)
 	if _, confirmed := fenceReadback(ev); confirmed && e.ec2Inventory != nil {

spinifex/operator/accountlifecycle/exercise.go

@@ -0,0 +1,207 @@
+package accountlifecycle
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
+	"github.com/nats-io/nats.go"
+)
+
+// PlatformCleaner is the EC2 platform cleanup the exercise runs.
+type PlatformCleaner = accountcleanup.PlatformCleaner
+
+// Exercise quiesces the account, cleans the EC2 platform kinds the evaluation
+// recorded as quiesced, then re-reads the owner inventory and holds it against
+// the cleanup. No legacy stage runs and the account record is never removed.
+func (e *Engine) Exercise(ctx context.Context, req Request) (*operatorv1.ExerciseResult, error) {
+	if _, err := e.checkDeletable(req.AccountID, req.AccountName); err != nil {
+		return nil, err
+	}
+	if req.Force {
+		return nil, errors.New("a non-finalizing run refuses force")
+	}
+	if e.Cleanup == nil || e.ec2Inventory == nil {
+		return nil, errors.New("exercise needs the platform cleanup and inventory")
+	}
+	generation, err := e.Accounts.QuiesceAccount(req.AccountID, req.OperationID, req.Fence)
+	if err != nil {
+		return nil, fmt.Errorf("quiesce account: %w", err)
+	}
+	quiescence := e.evaluateQuiescence(ctx, req, generation, true)
+	if e.OnQuiesced != nil {
+		e.OnQuiesced(ctx, quiescence)
+	}
+
+	result := &operatorv1.ExerciseResult{}
+	if len(quiescence.QuiescedScope) == 0 {
+		result.Skipped = operatorv1.ExerciseSkippedNoQuiescedKind
+	} else {
+		cleaned, err := e.Cleanup.CleanupPlatform(ctx, &accountcleanup.Request{
+			AccountID: req.AccountID, CorrelationRef: req.OperationID, Kinds: quiescence.QuiescedScope,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("platform cleanup: %w", err)
+		}
+		result.Cleanup = platformCleanup(cleaned)
+		result.Accounting = e.accountForRemainder(ctx, req, quiescence.QuiescedScope, result.Cleanup)
+	}
+	if e.OnExercised != nil {
+		e.OnExercised(ctx, *result)
+	}
+	return result, nil
+}
+
+// accountForRemainder re-reads the owner inventory after cleanup and holds it
+// against the cleanup result, per quiesced kind.
+func (e *Engine) accountForRemainder(ctx context.Context, req Request, kinds []string, cleanup *operatorv1.PlatformCleanup) []operatorv1.KindAccounting {
+	budget := ec2PlatformShadowBudget
+	if e.ec2PlatformShadowBudget > 0 {
+		budget = e.ec2PlatformShadowBudget
+	}
+	rereadCtx, cancel := context.WithTimeout(ctx, budget)
+	defer cancel()
+	reply := e.ec2Inventory.PlatformInventory(rereadCtx, req.AccountID, req.OperationID)
+	return AccountForRemainder(kinds, &reply, cleanup)
+}
+
+// AccountForRemainder marks a kind accounted only when its re-read was complete
+// and every record still there is named in the cleanup result with a reason.
+// A record the cleanup reported deleted that is back is unnamed.
+func AccountForRemainder(kinds []string, reread *ec2v1.PlatformInventoryReply, cleanup *operatorv1.PlatformCleanup) []operatorv1.KindAccounting {
+	out := make([]operatorv1.KindAccounting, 0, len(kinds))
+	for _, kind := range kinds {
+		entry := operatorv1.KindAccounting{Kind: kind, Status: operatorv1.AccountingAccounted}
+		status, complete := kindInventory(reread, kind)
+		if !complete {
+			entry.Status, entry.Reason = operatorv1.AccountingUnaccounted, operatorv1.AccountingReasonRereadIncomplete
+			out = append(out, entry)
+			continue
+		}
+		if status == ec2v1.PlatformStatusRecordsFound {
+			unnamed := false
+			for _, item := range reread.Items {
+				if item.Kind != kind {
+					continue
+				}
+				entry.Remaining++
+				unnamed = unnamed || !namedItem(cleanup, item.ResourceID, kind)
+			}
+			for _, count := range reread.Counts {
+				if count.Kind != kind {
+					continue
+				}
+				entry.Remaining += count.Count
+				unnamed = unnamed || !namedCount(cleanup, count)
+			}
+			if unnamed {
+				entry.Status, entry.Reason = operatorv1.AccountingUnaccounted, operatorv1.AccountingReasonUnnamedRemainder
+			}
+		}
+		out = append(out, entry)
+	}
+	return out
+}
+
+func namedItem(cleanup *operatorv1.PlatformCleanup, id, kind string) bool {
+	if cleanup == nil {
+		return false
+	}
+	for _, item := range cleanup.Items {
+		if item.ResourceID == id && item.Kind == kind && item.Disposition != accountcleanup.DispositionDeleted && item.Reason != "" {
+			return true
+		}
+	}
+	return false
+}
+
+func namedCount(cleanup *operatorv1.PlatformCleanup, want ec2v1.PlatformRecordCount) bool {
+	if cleanup == nil {
+		return false
+	}
+	for _, count := range cleanup.Counts {
+		if count.Kind == want.Kind && count.Class == want.Class && count.Disposition != accountcleanup.DispositionDeleted &&
+			count.Reason != "" && count.Count >= want.Count {
+			return true
+		}
+	}
+	return false
+}
+
+func platformCleanup(res *accountcleanup.Result) *operatorv1.PlatformCleanup {
+	out := &operatorv1.PlatformCleanup{Kinds: []operatorv1.CleanupKind{}}
+	for _, kind := range res.Kinds {
+		out.Kinds = append(out.Kinds, operatorv1.CleanupKind{Kind: kind.Kind, Status: kind.Status, IncompleteReasons: kind.IncompleteReasons})
+	}
+	for _, item := range res.Items {
+		out.Items = append(out.Items, operatorv1.CleanupItem{
+			ResourceID: item.ResourceID, Kind: item.Kind, Disposition: item.Disposition, Reason: item.Reason, MemberCount: item.MemberCount,
+		})
+	}
+	for _, count := range res.Counts {
+		out.Counts = append(out.Counts, operatorv1.CleanupCount{
+			Kind: count.Kind, Class: count.Class, Disposition: count.Disposition, Reason: count.Reason, Count: count.Count,
+		})
+	}
+	return out
+}
+
+// InstanceReferenceReader reads the key-pair and placement-group names the
+// account's non-terminated instances carry. It fails closed: a listing that is
+// not complete from every node is an error, never a partial answer.
+type InstanceReferenceReader struct {
+	nc      *nats.Conn
+	nodeIDs []string
+	budget  time.Duration
+}
+
+var _ accountcleanup.InstanceReferenceReader = (*InstanceReferenceReader)(nil)
+
+// NewInstanceReferenceReader builds the reader over the configured node set.
+func NewInstanceReferenceReader(nc *nats.Conn, nodeIDs []string) *InstanceReferenceReader {
+	return &InstanceReferenceReader{nc: nc, nodeIDs: nodeIDs, budget: ec2PlatformShadowBudget}
+}
+
+func (r *InstanceReferenceReader) AccountInstanceReferences(ctx context.Context, accountID string) (*accountcleanup.References, error) {
+	ctx, cancel := context.WithTimeout(ctx, r.budget)
+	defer cancel()
+	out, err := ec2instanceapi.DescribeAccountInstancesComplete(ctx, r.nc, r.nodeIDs, accountID)
+	if err != nil {
+		return nil, err
+	}
+	return instanceReferences(out), nil
+}
+
+func instanceReferences(out *ec2.DescribeInstancesOutput) *accountcleanup.References {
+	refs := &accountcleanup.References{
+		KeyNames: map[string]bool{}, PlacementGroupNames: map[string]bool{}, PlacementGroupIDs: map[string]bool{},
+	}
+	for _, reservation := range out.Reservations {
+		for _, instance := range reservation.Instances {
+			if instance == nil {
+				continue
+			}
+			if instance.State != nil && instance.State.Name != nil && terminalInstanceStates[*instance.State.Name] {
+				continue
+			}
+			if instance.KeyName != nil && *instance.KeyName != "" {
+				refs.KeyNames[*instance.KeyName] = true
+			}
+			if instance.Placement != nil {
+				if instance.Placement.GroupName != nil && *instance.Placement.GroupName != "" {
+					refs.PlacementGroupNames[*instance.Placement.GroupName] = true
+				}
+				if instance.Placement.GroupId != nil && *instance.Placement.GroupId != "" {
+					refs.PlacementGroupIDs[*instance.Placement.GroupId] = true
+				}
+			}
+		}
+	}
+	return refs
+}

spinifex/operator/accountlifecycle/exercise_test.go

@@ -0,0 +1,288 @@
+package accountlifecycle
+
+//test:in-package — the exercise's inventory and cleanup collaborators are
+// unexported seams, and the S7 invariants drive the engine directly.
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type stubCleaner struct {
+	calls  []accountcleanup.Request
+	result *accountcleanup.Result
+	err    error
+	// onCall runs inside the cleanup, so a test can observe what was recorded first.
+	onCall func()
+}
+
+func (s *stubCleaner) CleanupPlatform(_ context.Context, req *accountcleanup.Request) (*accountcleanup.Result, error) {
+	s.calls = append(s.calls, *req)
+	if s.onCall != nil {
+		s.onCall()
+	}
+	return s.result, s.err
+}
+
+// sequencedInventory answers the quiescence evaluation first, then the re-read.
+type sequencedInventory struct {
+	replies []ec2v1.PlatformInventoryReply
+	calls   int
+}
+
+func (s *sequencedInventory) PlatformInventory(context.Context, string, string) ec2v1.PlatformInventoryReply {
+	reply := s.replies[min(s.calls, len(s.replies)-1)]
+	s.calls++
+	return reply
+}
+
+func exerciseEngine(accounts AccountStore, cleaner PlatformCleaner, inventory ec2PlatformInventory) *Engine {
+	engine := quietEngine(accounts)
+	engine.Cleanup = cleaner
+	engine.ec2Inventory = inventory
+	return engine
+}
+
+func exerciseRequest() Request {
+	req := request()
+	req.OperationID, req.Fence = "op-1", 1
+	return req
+}
+
+func cleanResult() *accountcleanup.Result {
+	res := &accountcleanup.Result{AccountID: testAccountID}
+	for _, kind := range ec2v1.PlatformKinds() {
+		res.Kinds = append(res.Kinds, accountcleanup.KindReport{Kind: kind, Status: ec2v1.PlatformStatusObservedClean})
+	}
+	return res
+}
+
+func accountingFor(result *operatorv1.ExerciseResult, kind string) operatorv1.KindAccounting {
+	for _, entry := range result.Accounting {
+		if entry.Kind == kind {
+			return entry
+		}
+	}
+	return operatorv1.KindAccounting{}
+}
+
+// The exercise records the evaluation before it cleans, cleans only the kinds that
+// evaluation recorded as quiesced, and never removes the account record.
+func TestExerciseCleansTheQuiescedKindsAfterRecordingAndKeepsTheAccount(t *testing.T) {
+	accounts := newFakeAccounts()
+	var recorded *operatorv1.IAMQuiescence
+	cleaner := &stubCleaner{result: cleanResult()}
+	cleaner.onCall = func() { assert.NotNil(t, recorded, "quiescence must be recorded before any cleanup") }
+	inventory := &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory()}}
+	engine := exerciseEngine(accounts, cleaner, inventory)
+	engine.OnQuiesced = func(_ context.Context, q operatorv1.IAMQuiescence) { recorded = &q }
+
+	result, err := engine.Exercise(context.Background(), exerciseRequest())
+
+	require.NoError(t, err)
+	require.NotNil(t, recorded)
+	assert.ElementsMatch(t, ec2PlatformKinds, recorded.QuiescedScope)
+	require.Len(t, cleaner.calls, 1)
+	assert.ElementsMatch(t, recorded.QuiescedScope, cleaner.calls[0].Kinds)
+	assert.Equal(t, "op-1", cleaner.calls[0].CorrelationRef)
+	assert.False(t, accounts.deleted, "an exercise never removes the account record")
+	assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
+	assert.Equal(t, 2, inventory.calls, "one evaluation read and one final re-read")
+	for _, kind := range recorded.QuiescedScope {
+		assert.Equal(t, operatorv1.AccountingAccounted, accountingFor(result, kind).Status, kind)
+	}
+}
+
+// The final re-read is held against the cleanup result and the quiescence record
+// is not rewritten from it.
+func TestExerciseAccountsForARemainderOnlyWhenTheCleanupNamedIt(t *testing.T) {
+	cases := map[string]struct {
+		cleanup *accountcleanup.Result
+		reread  ec2v1.PlatformInventoryReply
+		want    string
+		reason  string
+	}{
+		"late write the cleanup never saw": {
+			cleanup: cleanResult(),
+			reread:  rereadWithKeyPair("key-late"),
+			want:    operatorv1.AccountingUnaccounted, reason: operatorv1.AccountingReasonUnnamedRemainder,
+		},
+		"cleanup deleted it but it is back": {
+			cleanup: withItem(cleanResult(), accountcleanup.Item{ResourceID: "key-1", Kind: ec2v1.PlatformKindKeyPair, Disposition: accountcleanup.DispositionDeleted}),
+			reread:  rereadWithKeyPair("key-1"),
+			want:    operatorv1.AccountingUnaccounted, reason: operatorv1.AccountingReasonUnnamedRemainder,
+		},
+		"cleanup blocked it with a reason": {
+			cleanup: withItem(cleanResult(), accountcleanup.Item{ResourceID: "key-1", Kind: ec2v1.PlatformKindKeyPair, Disposition: accountcleanup.DispositionBlocked, Reason: accountcleanup.ReasonInstanceReference}),
+			reread:  rereadWithKeyPair("key-1"),
+			want:    operatorv1.AccountingAccounted,
+		},
+		"re-read incomplete": {
+			cleanup: cleanResult(),
+			reread:  *incompleteKeyPairs(),
+			want:    operatorv1.AccountingUnaccounted, reason: operatorv1.AccountingReasonRereadIncomplete,
+		},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			accounts := newFakeAccounts()
+			inventory := &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory(), tc.reread}}
+			engine := exerciseEngine(accounts, &stubCleaner{result: tc.cleanup}, inventory)
+			var recorded operatorv1.IAMQuiescence
+			engine.OnQuiesced = func(_ context.Context, q operatorv1.IAMQuiescence) { recorded = q }
+
+			result, err := engine.Exercise(context.Background(), exerciseRequest())
+
+			require.NoError(t, err)
+			entry := accountingFor(result, ec2v1.PlatformKindKeyPair)
+			assert.Equal(t, tc.want, entry.Status)
+			assert.Equal(t, tc.reason, entry.Reason)
+			assert.Contains(t, recorded.QuiescedScope, ec2v1.PlatformKindKeyPair, "quiescence is not rewritten from the re-read")
+		})
+	}
+}
+
+func rereadWithKeyPair(id string) ec2v1.PlatformInventoryReply {
+	reply := *completeInventory()
+	reply.Kinds[0].Status = ec2v1.PlatformStatusRecordsFound
+	reply.Items = []ec2v1.PlatformItem{{ResourceID: id, Kind: ec2v1.PlatformKindKeyPair, Disposition: accountcleanup.DispositionBlocked}}
+	return reply
+}
+
+func incompleteKeyPairs() *ec2v1.PlatformInventoryReply {
+	reply := completeInventory()
+	reply.Kinds[0].Status = ec2v1.PlatformStatusScanIncomplete
+	return reply
+}
+
+func withItem(res *accountcleanup.Result, item accountcleanup.Item) *accountcleanup.Result {
+	res.Items = append(res.Items, item)
+	return res
+}
+
+// With no kind quiesced the exercise runs no cleanup and records why.
+func TestExerciseWithNoQuiescedKindRunsNoCleanup(t *testing.T) {
+	accounts := newFakeAccounts()
+	cleaner := &stubCleaner{result: cleanResult()}
+	incomplete := ec2v1.IncompletePlatformInventory(testAccountID, "op-1", ec2v1.PlatformReasonTimeout)
+	engine := exerciseEngine(accounts, cleaner, &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{incomplete}})
+
+	result, err := engine.Exercise(context.Background(), exerciseRequest())
+
+	require.NoError(t, err)
+	assert.Empty(t, cleaner.calls)
+	assert.Equal(t, operatorv1.ExerciseSkippedNoQuiescedKind, result.Skipped)
+	assert.Nil(t, result.Cleanup)
+}
+
+func TestExerciseRefusesForceAndAMissingCollaborator(t *testing.T) {
+	accounts := newFakeAccounts()
+	req := exerciseRequest()
+	req.Force = true
+	_, err := exerciseEngine(accounts, &stubCleaner{}, &stubInventory{}).Exercise(context.Background(), req)
+	require.Error(t, err)
+
+	_, err = exerciseEngine(accounts, nil, &stubInventory{}).Exercise(context.Background(), exerciseRequest())
+	require.Error(t, err)
+	assert.Empty(t, accounts.quiesces, "a refused exercise quiesces nothing")
+}
+
+// ADR-0003:S7: the exercise is an automated actor that deletes customer-visible
+// records, so it acts only on kinds whose quiescence was recorded under a confirmed
+// fence, and a failed cleanup leaves the account where it is.
+func TestS7_ExerciseCleansNothingUnlessItsQuiescenceWasRecordedUnderAConfirmedFence(t *testing.T) {
+	cases := map[string]func(*Engine, *fakeAccounts) *sequencedInventory{
+		"read-back fails": func(e *Engine, a *fakeAccounts) *sequencedInventory {
+			e.Accounts = &readbackFails{fakeAccounts: a}
+			return &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory()}}
+		},
+		"inventory incomplete": func(*Engine, *fakeAccounts) *sequencedInventory {
+			return &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{
+				ec2v1.IncompletePlatformInventory(testAccountID, "op-1", ec2v1.PlatformReasonNoResponder),
+			}}
+		},
+	}
+	for name, setup := range cases {
+		t.Run(name, func(t *testing.T) {
+			accounts := newFakeAccounts()
+			cleaner := &stubCleaner{result: cleanResult()}
+			engine := exerciseEngine(accounts, cleaner, nil)
+			engine.ec2Inventory = setup(engine, accounts)
+
+			_, err := engine.Exercise(context.Background(), exerciseRequest())
+
+			require.NoError(t, err)
+			assert.Empty(t, cleaner.calls, "no kind may be cleaned without a recorded quiescence")
+			assert.False(t, accounts.deleted)
+		})
+	}
+
+	t.Run("only the kinds with a complete inventory are cleaned", func(t *testing.T) {
+		accounts := newFakeAccounts()
+		cleaner := &stubCleaner{result: cleanResult()}
+		partial := completeInventory()
+		partial.Kinds[1].Status = ec2v1.PlatformStatusScanIncomplete
+		engine := exerciseEngine(accounts, cleaner, &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*partial}})
+
+		_, err := engine.Exercise(context.Background(), exerciseRequest())
+
+		require.NoError(t, err)
+		require.Len(t, cleaner.calls, 1)
+		assert.ElementsMatch(t, []string{ec2v1.PlatformKindKeyPair, ec2v1.PlatformKindLaunchTemplate}, cleaner.calls[0].Kinds)
+	})
+
+	t.Run("a failed cleanup returns an error and removes nothing", func(t *testing.T) {
+		accounts := newFakeAccounts()
+		engine := exerciseEngine(accounts, &stubCleaner{err: errors.New("owner down")}, &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory()}})
+
+		_, err := engine.Exercise(context.Background(), exerciseRequest())
+
+		require.Error(t, err)
+		assert.False(t, accounts.deleted)
+	})
+}
+
+// ADR-0003:S7: an ordinary teardown never runs the platform cleanup and never
+// records a quiesced scope, whatever its inventory says.
+func TestS7_OrdinaryTeardownNeverCleansOrRecordsAScope(t *testing.T) {
+	accounts := newFakeAccounts()
+	cleaner := &stubCleaner{result: cleanResult()}
+	engine := exerciseEngine(accounts, cleaner, &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory()}})
+	var recorded operatorv1.IAMQuiescence
+	engine.OnQuiesced = func(_ context.Context, q operatorv1.IAMQuiescence) { recorded = q }
+
+	_, err := engine.Teardown(context.Background(), exerciseRequest())
+
+	require.NoError(t, err)
+	assert.Empty(t, cleaner.calls)
+	assert.Empty(t, recorded.QuiescedScope)
+}
+
+func TestInstanceReferencesSkipTerminalInstancesAndNameKeysAndGroups(t *testing.T) {
+	refs := instanceReferences(&ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{
+		instanceWith("running", "kp-a", "pg-a", "pg-1"),
+		instanceWith("stopped", "kp-b", "", ""),
+		instanceWith("terminated", "kp-gone", "pg-gone", "pg-9"),
+	}}}})
+
+	assert.Equal(t, map[string]bool{"kp-a": true, "kp-b": true}, refs.KeyNames)
+	assert.Equal(t, map[string]bool{"pg-a": true}, refs.PlacementGroupNames)
+	assert.Equal(t, map[string]bool{"pg-1": true}, refs.PlacementGroupIDs)
+}
+
+func instanceWith(state, key, groupName, groupID string) *ec2.Instance {
+	return &ec2.Instance{
+		State:     &ec2.InstanceState{Name: aws.String(state)},
+		KeyName:   aws.String(key),
+		Placement: &ec2.Placement{GroupName: aws.String(groupName), GroupId: aws.String(groupID)},
+	}
+}

spinifex/operator/accountlifecycle/quiescence.go

@@ -7,17 +7,17 @@ import (
 	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
 )
 
-// finalInventoryRereadWired says whether anything re-reads the owner inventory
-// before the account record goes. Until it does, a write landing after the
-// reaper stage is orphaned, so no kind may be recorded as quiesced.
-var finalInventoryRereadWired = false
-
 // QuiesceEvidence is what the engine read after QuiesceAccount returned.
 type QuiesceEvidence struct {
 	OperationID string
 	Fence       uint64
 	Generation  uint64
 
+	// Exercise is set only for a non-finalizing run, which re-reads the owner
+	// inventory after its cleanup and removes no account record. An ordinary
+	// delete re-reads nothing before the record goes, so it records no scope.
+	Exercise bool
+
 	// Account is the IAM record read back after the quiesce.
 	Account     *Account
 	ReadbackErr error
@@ -79,7 +79,8 @@ var otherScopes = []unresolvedEntry{
 
 // EvaluateQuiescence applies ADR-0008:S9's three conditions to the evidence and
 // records what holds. A fence the read-back does not confirm stops everything
-// else; otherwise every scope is listed, and none is reported quiesced yet.
+// else; otherwise every scope is listed. A kind is quiesced only for an
+// exercise whose owner inventory for it was complete.
 func EvaluateQuiescence(ev QuiesceEvidence) operatorv1.IAMQuiescence {
 	out := operatorv1.IAMQuiescence{
 		OperationID:   ev.OperationID,
@@ -109,7 +110,7 @@ func EvaluateQuiescence(ev QuiesceEvidence) operatorv1.IAMQuiescence {
 		switch {
 		case !complete:
 			out.Unresolved = append(out.Unresolved, writerEntry("ec2-"+kind, operatorv1.ReasonOwnerInventoryIncomplete))
-		case !finalInventoryRereadWired:
+		case !ev.Exercise:
 			out.Unresolved = append(out.Unresolved, writerEntry("ec2-"+kind, operatorv1.ReasonFinalInventoryRereadNotWired))
 		default:
 			out.QuiescedScope = append(out.QuiescedScope, kind)

spinifex/operator/accountlifecycle/quiescence_test.go

@@ -72,12 +72,10 @@ func TestEvaluateQuiescenceRecordsEvidenceButQuiescesNothingWithoutAFinalReread(
 	assert.Equal(t, "2026-11-02T00:00:00Z", q.EvaluatedAt)
 }
 
-// The wiring step flips this switch; the complete kinds are then, and only then, named.
+// Only an exercise, which re-reads the inventory after cleanup, names the complete kinds.
 func TestEvaluateQuiescenceNamesCompleteKindsOnceTheRereadIsWired(t *testing.T) {
-	finalInventoryRereadWired = true
-	t.Cleanup(func() { finalInventoryRereadWired = false })
-
 	ev := confirmedEvidence()
+	ev.Exercise = true
 	ev.Inventory.Kinds[1].Status = ec2v1.PlatformStatusScanIncomplete
 	q := EvaluateQuiescence(ev)
 
@@ -133,9 +131,8 @@ func TestEvaluateQuiescenceRecordsNothingWhenTheFenceIsNotConfirmed(t *testing.T
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
-			finalInventoryRereadWired = true
-			t.Cleanup(func() { finalInventoryRereadWired = false })
 			ev := confirmedEvidence()
+			ev.Exercise = true
 			tc.mutate(&ev)
 			q := EvaluateQuiescence(ev)
 
@@ -172,11 +169,11 @@ func TestEveryOtherScopeIsRecordedUnresolvedByName(t *testing.T) {
 
 // Predastore has no accepted enforcement contract, so the scopes it admits writes
 // for stay unresolved and it is never cited as a consumer, even once the
-// re-read is wired and every L09 kind is complete.
+// run is an exercise and every L09 kind is complete.
 func TestPredastoreIsNeverCitedAndItsScopesNeverQuiesce(t *testing.T) {
-	finalInventoryRereadWired = true
-	t.Cleanup(func() { finalInventoryRereadWired = false })
-	q := EvaluateQuiescence(confirmedEvidence())
+	ev := confirmedEvidence()
+	ev.Exercise = true
+	q := EvaluateQuiescence(ev)
 
 	for _, scope := range []string{"s3", "ecr"} {
 		got := unresolvedFor(q, scope)

spinifex/operator/admin/account_delete.go

@@ -66,7 +66,10 @@ type accountDeletionJob struct {
 	ClientToken string `json:"clientToken"`
 	State       string `json:"state"`
 	Force       bool   `json:"force"`
-	StartedAt   string `json:"startedAt"`
+	// NonFinalizing marks an exercise: it quiesces and cleans the quiesced EC2
+	// platform kinds, then ends EXERCISED with the account still TERMINATING.
+	NonFinalizing bool   `json:"nonFinalizing,omitempty"`
+	StartedAt     string `json:"startedAt"`
 	// UpdatedAt is the running job's heartbeat. Its absence for long enough is
 	// what lets another gateway take over after a restart.
 	UpdatedAt  string                        `json:"updatedAt"`
@@ -88,6 +91,8 @@ type accountDeletionJob struct {
 	// IAM is the quiescence evaluation made under the current fence. A takeover
 	// starts without one and the new holder records its own.
 	IAM *operatorv1.IAMQuiescence `json:"iam,omitempty"`
+	// Exercise is what a non-finalizing run did after quiescence.
+	Exercise *operatorv1.ExerciseResult `json:"exercise,omitempty"`
 }
 
 // deletionTakeover records one change of holder.
@@ -99,8 +104,9 @@ type deletionTakeover struct {
 }
 
 const (
-	takeoverReasonLeaseExpired = "lease_expired"
-	takeoverReasonFailed       = "failed"
+	takeoverReasonLeaseExpired     = "lease_expired"
+	takeoverReasonFailed           = "failed"
+	takeoverReasonExerciseFinished = "exercise_finished"
 )
 
 // deleteAccount starts a teardown, or answers a dry run inline.
@@ -118,6 +124,11 @@ func (d Deps) deleteAccount(ctx context.Context, body []byte) (any, error) {
 		return nil, errors.New(awserrors.ErrorInternalError)
 	}
 
+	if req.NonFinalizing && d.PlatformCleaner == nil {
+		slog.Error("DeleteAccount: platform cleanup not available")
+		return nil, errors.New(awserrors.ErrorServiceUnavailable)
+	}
+
 	engine, err := accountteardown.NewClusterEngine(ctx, d.NATS, accountteardown.ClusterNodes{Expected: d.ExpectedNodes, DaemonIDs: d.NodeIDs, GatewayIDs: d.GatewayNodeIDs}, d.Termination, d.BucketStore)
 	if err != nil {
 		slog.Error("DeleteAccount: failed to build teardown engine", "err", err)
@@ -262,6 +273,10 @@ func (d Deps) runAccountDeletion(
 	tracker := &deletionTracker{jobs: jobs, job: job, revision: revision, cancel: cancel}
 	engine.OnStage = tracker.stageFinished
 	engine.OnQuiesced = tracker.quiesced
+	if job.NonFinalizing {
+		engine.Cleanup = d.PlatformCleaner
+		engine.OnExercised = tracker.exercised
+	}
 
 	go func() {
 		defer cancel()
@@ -269,13 +284,19 @@ func (d Deps) runAccountDeletion(
 		stop := tracker.startHeartbeat(ctx)
 		defer stop()
 
-		result, err := engine.Teardown(ctx, accountteardown.Request{
+		request := accountteardown.Request{
 			AccountID:   job.AccountID,
 			AccountName: job.AccountName,
 			Force:       job.Force,
 			OperationID: job.OperationID,
 			Fence:       job.Fence,
-		})
+		}
+		if job.NonFinalizing {
+			_, err := engine.Exercise(ctx, request)
+			tracker.finished(ctx, nil, err)
+			return
+		}
+		result, err := engine.Teardown(ctx, request)
 		tracker.finished(ctx, result, err)
 	}()
 }
@@ -308,6 +329,13 @@ func (t *deletionTracker) quiesced(ctx context.Context, quiescence operatorv1.IA
 	t.persist(ctx)
 }
 
+func (t *deletionTracker) exercised(ctx context.Context, result operatorv1.ExerciseResult) {
+	t.mu.Lock()
+	t.job.Exercise = &result
+	t.mu.Unlock()
+	t.persist(ctx)
+}
+
 func (t *deletionTracker) finished(ctx context.Context, result *accountteardown.Result, err error) {
 	t.mu.Lock()
 	if result != nil {
@@ -316,6 +344,8 @@ func (t *deletionTracker) finished(ctx context.Context, result *accountteardown.
 	if err != nil {
 		t.job.State = operatorv1.DeletionStateFailed
 		t.job.Error = err.Error()
+	} else if t.job.NonFinalizing {
+		t.job.State = operatorv1.DeletionStateExercised
 	} else {
 		t.job.State = operatorv1.DeletionStateCompleted
 	}
@@ -445,8 +475,10 @@ func claimDeletionJob(
 		StartedAt:   now.Format(time.RFC3339),
 		UpdatedAt:   now.Format(time.RFC3339),
 		Fence:       1,
-		Holder:      holder,
-		LeaseUntil:  now.Add(accountDeletionStale).Format(time.RFC3339),
+
+		NonFinalizing: req.NonFinalizing,
+		Holder:        holder,
+		LeaseUntil:    now.Add(accountDeletionStale).Format(time.RFC3339),
 	}
 	record, err := json.Marshal(job)
 	if err != nil {
@@ -478,6 +510,9 @@ func claimDeletionJob(
 	}
 
 	reason := takeoverReasonFailed
+	if existing.State == operatorv1.DeletionStateExercised {
+		reason = takeoverReasonExerciseFinished
+	}
 	if existing.State == operatorv1.DeletionStateRunning {
 		if !deletionLeaseExpired(existing, now) {
 			return nil, 0, nil, errors.New(awserrors.ErrorOperationInProgress)
@@ -512,8 +547,10 @@ func takeOverDeletionJob(
 		Force:       req.Force,
 		StartedAt:   now.Format(time.RFC3339),
 		UpdatedAt:   now.Format(time.RFC3339),
-		Holder:      holder,
-		LeaseUntil:  now.Add(accountDeletionStale).Format(time.RFC3339),
+
+		NonFinalizing: req.NonFinalizing,
+		Holder:        holder,
+		LeaseUntil:    now.Add(accountDeletionStale).Format(time.RFC3339),
 	}
 	if existing.OperationID != "" {
 		job.OperationID = existing.OperationID
@@ -557,7 +594,7 @@ func replayDeletionJob(
 		return nil, err
 	}
 	if existing.ClientToken == req.ClientToken {
-		if existing.AccountName != req.AccountName {
+		if existing.AccountName != req.AccountName || existing.NonFinalizing != req.NonFinalizing {
 			return nil, errors.New(awserrors.ErrorIdempotentParameterMismatch)
 		}
 		// A RUNNING job whose lease has lapsed has no holder to follow, so the
@@ -703,6 +740,9 @@ func validateDeleteAccountRequest(req *operatorv1.DeleteAccountRequest) error {
 	if !accountIDRE.MatchString(req.AccountID) {
 		return errors.New(awserrors.ErrorInvalidParameterValue)
 	}
+	if req.NonFinalizing && (req.Force || req.DryRun) {
+		return errors.New(awserrors.ErrorInvalidParameterCombination)
+	}
 	if req.DryRun {
 		// A dry run deletes nothing, so it needs neither the name confirmation
 		// nor a token to make it idempotent.

spinifex/operator/admin/account_delete_exercise_test.go

@@ -0,0 +1,133 @@
+package admin
+
+//test:in-package — the exercise runs through the job record, the takeover and
+// the resumer, all of which are unexported.
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const exerciseToken = "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d"
+
+type panicCleaner struct{ calls int }
+
+func (p *panicCleaner) CleanupPlatform(context.Context, *accountcleanup.Request) (*accountcleanup.Result, error) {
+	p.calls++
+	return &accountcleanup.Result{}, nil
+}
+
+func exerciseBody(t *testing.T, accountID, name, token string, mutate func(*operatorv1.DeleteAccountRequest)) []byte {
+	t.Helper()
+	req := operatorv1.DeleteAccountRequest{AccountID: accountID, AccountName: name, ClientToken: token, NonFinalizing: true}
+	if mutate != nil {
+		mutate(&req)
+	}
+	body, err := json.Marshal(req)
+	require.NoError(t, err)
+	return body
+}
+
+func awaitState(t *testing.T, gw Deps, accountID, state string) *accountDeletionJob {
+	t.Helper()
+	require.Eventually(t, func() bool {
+		return describeDeletion(t, gw, accountID).State == state
+	}, 30*time.Second, 100*time.Millisecond)
+	return describeDeletion(t, gw, accountID)
+}
+
+// An exercise ends EXERCISED with the account still TERMINATING, and a later
+// ordinary delete takes the same operation over to finish it.
+func TestNonFinalizingDeleteEndsExercisedAndALaterDeleteTakesItOver(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	gw.PlatformCleaner = &panicCleaner{}
+	account := newTenantAccount(t, gw, "tenant@example.com")
+
+	_, err := gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken, nil))
+	require.NoError(t, err)
+
+	exercised := awaitState(t, gw, account.AccountID, operatorv1.DeletionStateExercised)
+	require.NotNil(t, exercised.IAM)
+	require.NotNil(t, exercised.Exercise)
+	assert.True(t, exercised.NonFinalizing)
+	assert.NotEmpty(t, exercised.FinishedAt)
+	assert.Empty(t, exercised.Stages, "no legacy stage runs")
+	stored, err := gw.IAM.GetAccount(account.AccountID)
+	require.NoError(t, err)
+	assert.Equal(t, "TERMINATING", stored.Status)
+
+	_, err = gw.deleteAccount(t.Context(), deleteAccountBody(t, account.AccountID, "tenant@example.com", deleteClientToken))
+	require.NoError(t, err)
+
+	done := awaitState(t, gw, account.AccountID, operatorv1.DeletionStateCompleted)
+	assert.Equal(t, exercised.OperationID, done.OperationID)
+	assert.Equal(t, uint64(2), done.Fence)
+	require.Len(t, done.Takeovers, 1)
+	assert.Equal(t, takeoverReasonExerciseFinished, done.Takeovers[0].Reason)
+	assert.Nil(t, done.Exercise, "the takeover starts a fresh run")
+	assert.Equal(t, uint64(2), done.IAM.Fence)
+	assert.Empty(t, done.IAM.QuiescedScope, "an ordinary delete records no scope")
+}
+
+func TestNonFinalizingIsPartOfTheIdempotencyKeyAndRefusesForce(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	gw.PlatformCleaner = &panicCleaner{}
+	account := newTenantAccount(t, gw, "tenant@example.com")
+
+	_, err := gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken,
+		func(r *operatorv1.DeleteAccountRequest) { r.Force = true }))
+	require.EqualError(t, err, awserrors.ErrorInvalidParameterCombination)
+
+	_, err = gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken, nil))
+	require.NoError(t, err)
+	awaitState(t, gw, account.AccountID, operatorv1.DeletionStateExercised)
+
+	_, err = gw.deleteAccount(t.Context(), deleteAccountBody(t, account.AccountID, "tenant@example.com", exerciseToken))
+	require.EqualError(t, err, awserrors.ErrorIdempotentParameterMismatch, "the same token cannot turn an exercise into a delete")
+
+	again, err := gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken, nil))
+	require.NoError(t, err)
+	assert.Equal(t, operatorv1.DeletionStateExercised, again.(*DeleteAccountResponse).State, "a repeated exercise is answered from the record")
+}
+
+func TestNonFinalizingWithoutACleanupIsRefusedBeforeAnyJobExists(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	account := newTenantAccount(t, gw, "tenant@example.com")
+
+	_, err := gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken, nil))
+
+	require.EqualError(t, err, awserrors.ErrorServiceUnavailable)
+	jobs, err := gw.accountDeletionStore(t.Context())
+	require.NoError(t, err)
+	stored, err := readDeletionJob(t.Context(), jobs, account.AccountID)
+	require.NoError(t, err)
+	assert.Nil(t, stored)
+}
+
+// A resumed exercise stays an exercise: the resumer must not turn a recorded
+// non-finalizing decision into a deletion.
+func TestResumerKeepsAnAbandonedExerciseNonFinalizing(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	gw.PlatformCleaner = &panicCleaner{}
+	account := newTenantAccount(t, gw, "tenant@example.com")
+	job := abandonedJob(account.AccountID, "tenant@example.com")
+	job.NonFinalizing = true
+	storeDeletionJob(t, gw, job)
+
+	require.Equal(t, 1, gw.resumeDeletions(t.Context(), time.Now().UTC()))
+
+	done := awaitState(t, gw, account.AccountID, operatorv1.DeletionStateExercised)
+	assert.Equal(t, uint64(2), done.Fence)
+	assert.True(t, done.NonFinalizing)
+	stored, err := gw.IAM.GetAccount(account.AccountID)
+	require.NoError(t, err)
+	assert.Equal(t, "TERMINATING", stored.Status, "the account record is never removed")
+}

spinifex/operator/admin/account_delete_resumer.go

@@ -110,6 +110,8 @@ func (d Deps) resumeDeletion(
 		AccountName: existing.AccountName,
 		ClientToken: existing.ClientToken,
 		Force:       existing.Force,
+
+		NonFinalizing: existing.NonFinalizing,
 	}
 	job, newRevision, _, err := takeOverDeletionJob(ctx, jobs, req, d.holderID(), existing, revision, takeoverReasonLeaseExpired, now)
 	if err != nil {

spinifex/operator/admin/account_delete_resumer_test.go

@@ -45,6 +45,7 @@ func TestS7_DeletionResumerActsOnlyOnAnAcceptedOperationWithoutALiveLease(t *tes
 		},
 		"failed job":    func(j *accountDeletionJob) { j.State = operatorv1.DeletionStateFailed },
 		"completed job": func(j *accountDeletionJob) { j.State = operatorv1.DeletionStateCompleted },
+		"exercised job": func(j *accountDeletionJob) { j.State = operatorv1.DeletionStateExercised },
 		"legacy record": func(j *accountDeletionJob) { j.OperationID, j.Fence, j.LeaseUntil = "", 0, "" },
 	}
 	for name, mutate := range cases {

spinifex/operator/admin/image_import.go

@@ -76,7 +76,7 @@ func ImportImage(ctx context.Context, provider ebsprovider.EBSProvider, opts Imp
 
 	// Unpublish regardless: a volume left published cannot be snapshotted or
 	// attached, and on the failure path it would strand the export too.
-	if err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
+	if _, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(),
 		VolumeID:  opts.VolumeID,
 		NodeID:    opts.NodeID,

spinifex/operator/admin/image_import_test.go

@@ -49,7 +49,7 @@ func (p *recordingProvider) PublishVolume(ctx context.Context, req ebsprovider.P
 	return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: export}, nil
 }
 
-func (p *recordingProvider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error {
+func (p *recordingProvider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
 	p.calls = append(p.calls, "unpublish")
 	return p.MemoryProvider.UnpublishVolume(ctx, req)
 }

spinifex/operator/admin/registration.go

@@ -58,7 +58,10 @@ type Deps struct {
 	IAM iamdomain.IAMService
 	// Termination is the concrete IAM service, the only holder of the capability
 	// to quiesce and remove an account. DeleteAccount needs it.
-	Termination       accountteardown.IAMBackend
+	Termination accountteardown.IAMBackend
+	// PlatformCleaner is the in-process EC2 platform cleanup a non-finalizing
+	// DeleteAccount runs. Nil refuses that request.
+	PlatformCleaner   accountteardown.PlatformCleaner
 	Quota             *quota.Service
 	ExpectedNodes     int
 	NodeIDs           []string

spinifex/operator/cli/admin_account_delete.go

@@ -61,6 +61,8 @@ func init() {
 	accountCmd.AddCommand(accountDeleteCmd)
 
 	accountDeleteCmd.Flags().Bool("dry-run", false, "Report what would be deleted and change nothing")
+	accountDeleteCmd.Flags().Bool("non-finalizing", false, "Quiesce and clean the EC2 platform kinds, then stop with the account still TERMINATING")
+	_ = accountDeleteCmd.Flags().MarkHidden("non-finalizing")
 	accountDeleteCmd.Flags().Bool("force", false, "Escalate past state guards for resources that will not delete")
 	accountDeleteCmd.Flags().Bool("yes", false, "Skip the interactive confirmation (requires --name)")
 	accountDeleteCmd.Flags().String("name", "", "Account name, which must match the stored record")

spinifex/operator/cli/admin_account_delete_remote.go

@@ -24,6 +24,7 @@ var accountDeletePollInterval = 10 * time.Second
 func runAccountDeleteRemote(cmd *cobra.Command, accountID string) error {
 	dryRun, _ := cmd.Flags().GetBool("dry-run")
 	force, _ := cmd.Flags().GetBool("force")
+	nonFinalizing, _ := cmd.Flags().GetBool("non-finalizing")
 	assumeYes, _ := cmd.Flags().GetBool("yes")
 	name, _ := cmd.Flags().GetString("name")
 	clientToken, _ := cmd.Flags().GetString("client-token")
@@ -66,6 +67,7 @@ func runAccountDeleteRemote(cmd *cobra.Command, accountID string) error {
 
 	started, err := deleteAccountRemote(ctx, target, operatorv1.DeleteAccountRequest{
 		AccountID: accountID, AccountName: name, ClientToken: clientToken, Force: force,
+		NonFinalizing: nonFinalizing,
 	})
 	if err != nil {
 		var adminErr *adminError
@@ -101,6 +103,9 @@ func followAccountDeletion(ctx context.Context, target adminTarget, accountID st
 		case operatorv1.DeletionStateCompleted:
 			fmt.Printf("\nAccount %s deleted.\n", accountID)
 			return nil
+		case operatorv1.DeletionStateExercised:
+			fmt.Printf("\nAccount %s exercised and left TERMINATING; a normal delete finishes it.\n", accountID)
+			return nil
 		case operatorv1.DeletionStateFailed:
 			fmt.Fprintf(os.Stderr, "\nThe account is still TERMINATING so its leftovers keep an owner.\n")
 			return fmt.Errorf("teardown failed: %s", job.Error)

spinifex/operator/cli/admin_test.go

@@ -1748,3 +1748,31 @@ func TestApplyNetworkConfig_PropagatesPoolDNSServersWithoutSelf(t *testing.T) {
 	require.Len(t, settings.Pools, 1)
 	assert.Equal(t, []string{"1.1.1.1", "10.0.0.6", "8.8.8.8"}, settings.Pools[0].DNSServers)
 }
+
+// A generated config writes no [nodes.<peer>.ebs] stanza, so a node switched to
+// a non-default provider must still load: an absent peer provider is unknown.
+func TestSpinifexTomlTemplate_NonDefaultProviderLoadsBesidePeers(t *testing.T) {
+	allNodes := map[string]formation.NodeInfo{
+		"node1": {Name: "node1", BindIP: "10.0.0.1", AdvertiseIP: "192.168.1.31"},
+		"node2": {Name: "node2", BindIP: "10.0.0.2", AdvertiseIP: "192.168.1.32"},
+		"node3": {Name: "node3", BindIP: "10.0.0.3", AdvertiseIP: "192.168.1.33"},
+	}
+	settings := multinodeNorthstarSettings("/etc/spinifex/northstar/northstar.toml")
+	settings.RemoteNodes = buildRemoteNodes(allNodes, "node1", settings.NorthstarConfigPath)
+	require.Len(t, settings.RemoteNodes, 2)
+
+	path := filepath.Join(t.TempDir(), "spinifex.toml")
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+	rendered, err := os.ReadFile(path)
+	require.NoError(t, err)
+	const defaultLine = "provider = \"viperblockd\"\n"
+	require.Equal(t, 1, strings.Count(string(rendered), defaultLine), "the template must set the provider for its own node only")
+	switched := strings.Replace(string(rendered), defaultLine, "provider = \"qemunbd\"\n", 1)
+	require.NoError(t, os.WriteFile(path, []byte(switched), 0o600))
+
+	cfg, err := config.LoadConfig(path)
+	require.NoError(t, err)
+	assert.Equal(t, config.EBSProviderQEMUNBD, cfg.Nodes["node1"].EBS.ResolvedProvider())
+	assert.Contains(t, cfg.Nodes, "node2")
+	assert.Contains(t, cfg.Nodes, "node3")
+}

spinifex/operator/cli/templates/spinifex.toml

@@ -136,6 +136,9 @@ encryption_key_file = "{{.EncryptionKeyFile}}"
 {{- end}}
 
 [nodes.{{.Node}}.ebs]
+# provider: every node in a cluster must name the same one, and the daemon
+# refuses to start on a provider other than the one the cluster recorded.
+# Changing it needs the cluster wiped and reformed.
 provider = "viperblockd"
 # Per-volume export tunables. Both are applied per nbdkit process and there is
 # one of those per volume, so the host cost of each is the value times the

spinifex/operator/upgrade/VERSIONS.md

@@ -21,6 +21,7 @@ Single source of truth for the current schema version of every config file Spini
 |---|---|---|
 | `spinifex-instance-state` | `5` | `daemon.InstanceStateBucketVersion` |
 | `spinifex-cluster-state` | `1` | `daemon.ClusterStateBucketVersion` |
+| `spinifex-cluster-config` | `1` | `daemon.ClusterConfigBucketVersion` |
 | `spinifex-terminated-instances` | `3` | `daemon.TerminatedInstanceBucketVersion` |
 
 ## Registered migrations

spinifex/providers/ebs/conformance/bench.go

@@ -137,7 +137,7 @@ func RunBenchSuite(b *testing.B, provider ebsprovider.EBSProvider, cfg BenchConf
 			}); err != nil {
 				b.Fatal(err)
 			}
-			if err := provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{
+			if _, err := provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{
 				Versioned: ebsprovider.NewVersioned(),
 				VolumeID:  volumeID,
 				NodeID:    cfg.node(),

spinifex/providers/ebs/conformance/conformance.go

@@ -6,6 +6,9 @@ package conformance
 import (
 	"bytes"
 	"context"
+	"errors"
+	"fmt"
+	"slices"
 	"strings"
 	"sync"
 	"testing"
@@ -34,18 +37,24 @@ var ReferenceCapabilities = ebsprovider.Capabilities{
 
 // capabilitiesOf reads what a provider advertises, so the suite can branch on
 // it the way the contract tells callers to.
-func capabilitiesOf(t *testing.T, provider ebsprovider.EBSProvider) ebsprovider.Capabilities {
+func capabilitiesOf(t *testing.T, provider ebsprovider.EBSProvider) *ebsprovider.GetCapabilitiesResponse {
 	t.Helper()
 	resp, err := provider.GetCapabilities(context.Background(), ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()})
 	require.NoError(t, err)
 	require.NotNil(t, resp)
-	return resp.Capabilities
+	return resp
 }
 
 // maxListPages bounds a drain so a provider whose token fails to advance
 // fails the test instead of hanging it.
 const maxListPages = 100
 
+// unpublish is UnpublishVolume for a case that asserts only on the error.
+func unpublish(ctx context.Context, provider ebsprovider.EBSProvider, req ebsprovider.UnpublishVolumeRequest) error {
+	_, err := provider.UnpublishVolume(ctx, req)
+	return err
+}
+
 // volumeIDs projects one page of refs down to its IDs.
 func volumeIDs(refs []ebsprovider.VolumeRef) []string {
 	ids := make([]string, 0, len(refs))
@@ -131,6 +140,11 @@ type SuiteConfig struct {
 	// also be published elsewhere. Empty skips that subtest, for a caller with
 	// only one node to offer.
 	OtherNodeID string
+
+	// KillExport ends the process serving a published export without telling
+	// the provider, standing in for a crash. Nil skips the dead-export case,
+	// for a provider whose export is not a separate process.
+	KillExport func(t *testing.T, published *ebsprovider.PublishedVolume)
 }
 
 func (c SuiteConfig) node() string {
@@ -176,7 +190,8 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 		return newTrackingProvider(t, newRawProvider(t))
 	}
 
-	capabilities := capabilitiesOf(t, newProvider(t))
+	advertised := capabilitiesOf(t, newProvider(t))
+	capabilities, contractVersions := advertised.Capabilities, advertised.ContractVersions
 
 	t.Run("GetCapabilities", func(t *testing.T) {
 		provider := newProvider(t)
@@ -193,6 +208,44 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 		assert.Equal(t, resp.Capabilities, again.Capabilities, "GetCapabilities must be stable across calls")
 	})
 
+	t.Run("ContractVersions", func(t *testing.T) {
+		t.Run("listed versions are ascending and include the legacy version", func(t *testing.T) {
+			require.NotEmpty(t, contractVersions, "a caller cannot choose a version for a provider that lists none")
+			assert.Contains(t, contractVersions, ebsprovider.LegacySchemaVersion,
+				"dropping the legacy version strands every caller still in an N/N-1 rollout")
+			for i := 1; i < len(contractVersions); i++ {
+				assert.Lessf(t, contractVersions[i-1], contractVersions[i], "contract versions %v are not strictly ascending", contractVersions)
+			}
+		})
+
+		t.Run("every listed version is answered in that version", func(t *testing.T) {
+			provider := newProvider(t)
+			for _, version := range contractVersions {
+				resp, err := provider.GetCapabilities(context.Background(), ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}})
+				require.NoErrorf(t, err, "version %d is listed as served", version)
+				assert.Equalf(t, version, resp.SchemaVersion, "a reply in another version than the request's is one an older caller may not parse")
+			}
+		})
+
+		t.Run("an unlisted version is refused without changing state", func(t *testing.T) {
+			provider := newProvider(t)
+			ctx := context.Background()
+			unknown := ebsprovider.Versioned{SchemaVersion: ebsprovider.SchemaVersion + 1}
+			_, err := provider.GetCapabilities(ctx, ebsprovider.GetCapabilitiesRequest{Versioned: unknown})
+			require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion)
+
+			volumeID := cfg.id("vol-ver-unknown")
+			_, err = provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: unknown, VolumeID: volumeID, NodeID: cfg.node()})
+			require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion)
+			// A refused publish that still exported the volume would leave it
+			// published, and a published volume cannot be deleted.
+			require.NoError(t, provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID}),
+				"a publish refused for its version must not have published the volume")
+		})
+	})
+
 	t.Run("CreateVolume", func(t *testing.T) {
 		t.Run("success", func(t *testing.T) {
 			provider := newProvider(t)
@@ -385,8 +438,6 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 	})
 
 	t.Run("ListVolumes", func(t *testing.T) {
-		capabilities := capabilitiesOf(t, newProvider(t))
-
 		t.Run("unsupported_capability when enumeration is not advertised", func(t *testing.T) {
 			if capabilities.VolumeEnumeration {
 				t.Skip("provider advertises volume enumeration")
@@ -694,8 +745,6 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 	})
 
 	t.Run("ListSnapshots", func(t *testing.T) {
-		capabilities := capabilitiesOf(t, newProvider(t))
-
 		t.Run("unsupported_capability when enumeration is not advertised", func(t *testing.T) {
 			if capabilities.SnapshotEnumeration {
 				t.Skip("provider advertises snapshot enumeration")
@@ -798,16 +847,23 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			}
 		})
 
+		// A republish succeeds at every listed version, so a caller at any of
+		// them gets the running export back rather than a refusal.
 		t.Run("idempotent republish to the same node", func(t *testing.T) {
-			provider := newProvider(t)
-			ctx := context.Background()
-			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-idem"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
-			require.NoError(t, err)
-			first, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-idem"), NodeID: cfg.node()})
-			require.NoError(t, err)
-			second, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-idem"), NodeID: cfg.node()})
-			require.NoError(t, err)
-			assert.Equal(t, first, second)
+			for _, version := range contractVersions {
+				t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) {
+					provider := newProvider(t)
+					ctx := context.Background()
+					volumeID := cfg.id(fmt.Sprintf("vol-pub-idem-v%d", version))
+					_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+					require.NoError(t, err)
+					first, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+					require.NoError(t, err)
+					second, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: volumeID, NodeID: cfg.node()})
+					require.NoError(t, err)
+					assert.Equal(t, first, second)
+				})
+			}
 		})
 
 		t.Run("volume_in_use when published to a different node", func(t *testing.T) {
@@ -824,6 +880,76 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse)
 		})
 
+		t.Run("held elsewhere is reported only from the version that defines it", func(t *testing.T) {
+			if cfg.otherNode() == "" {
+				t.Skip("no second node configured")
+			}
+			for _, version := range contractVersions {
+				t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) {
+					provider := newProvider(t)
+					ctx := context.Background()
+					volumeID := cfg.id(fmt.Sprintf("vol-pub-held-v%d", version))
+					_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+					require.NoError(t, err)
+					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+					require.NoError(t, err)
+
+					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: volumeID, NodeID: cfg.otherNode()})
+					require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse, "every version must still read the refusal as in use")
+					if version >= ebsprovider.HeldElsewhereSchemaVersion {
+						assert.ErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere)
+					} else {
+						assert.NotErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere,
+							"a version %d caller predates held_elsewhere and must get the volume_in_use it has always got", version)
+					}
+				})
+			}
+		})
+
+		t.Run("a republish in the other access mode is in use, not held elsewhere", func(t *testing.T) {
+			if !capabilities.ReadOnlyPublish {
+				t.Skip("read-only publish is not advertised")
+			}
+			for _, version := range contractVersions {
+				t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) {
+					provider := newProvider(t)
+					ctx := context.Background()
+					volumeID := cfg.id(fmt.Sprintf("vol-pub-mode-v%d", version))
+					_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+					require.NoError(t, err)
+					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+					require.NoError(t, err)
+
+					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: volumeID, NodeID: cfg.node(), ReadOnly: true})
+					require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse)
+					assert.NotErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere,
+						"this node holds the volume, so a caller told it is held elsewhere would wait for a node that does not exist")
+				})
+			}
+		})
+
+		t.Run("a republish after the export died returns a serving export", func(t *testing.T) {
+			if cfg.KillExport == nil {
+				t.Skip("no way to kill this provider's export")
+			}
+			RequireNBDTools(t)
+			provider := newProvider(t)
+			ctx := context.Background()
+			volumeID := cfg.id("vol-pub-dead")
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			first, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+			firstURI := dialURI(t, publishedNBDURI(t, first))
+
+			cfg.KillExport(t, first)
+			require.False(t, nbdReachable(firstURI), "the export still answers, so the kill did not take and this case proves nothing")
+
+			second, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err, "a dead export must not make its volume unpublishable")
+			nbdInfoRetry(t, dialURI(t, publishedNBDURI(t, second)))
+		})
+
 		t.Run("not_found on absent volume", func(t *testing.T) {
 			provider := newProvider(t)
 			_, err := provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed"), NodeID: cfg.node()})
@@ -845,21 +971,105 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			require.NoError(t, err)
 			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-unpub-ok"), NodeID: cfg.node()})
 			require.NoError(t, err)
-			require.NoError(t, provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-unpub-ok"), NodeID: cfg.node()}))
+			require.NoError(t, unpublish(ctx, provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-unpub-ok"), NodeID: cfg.node()}))
 		})
 
 		t.Run("absent target is idempotent", func(t *testing.T) {
 			provider := newProvider(t)
-			require.NoError(t, provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed"), NodeID: cfg.node()}))
+			require.NoError(t, unpublish(context.Background(), provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed"), NodeID: cfg.node()}))
+		})
+
+		t.Run("torn down reports whether an export was removed", func(t *testing.T) {
+			if !slices.Contains(contractVersions, ebsprovider.NoSealSchemaVersion) {
+				t.Skipf("torn_down is defined from version %d, which the provider does not list", ebsprovider.NoSealSchemaVersion)
+			}
+			provider := newProvider(t)
+			ctx := context.Background()
+			volumeID := cfg.id("vol-unpub-torn")
+			v3 := ebsprovider.Versioned{SchemaVersion: ebsprovider.NoSealSchemaVersion}
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+
+			first, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: v3, VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+			assert.True(t, first.TornDown, "the volume was published, so this unpublish removed an export")
+			again, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: v3, VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+			assert.False(t, again.TornDown, "nothing was left to remove, so a retry must not claim it removed something")
+		})
+
+		t.Run("no_seal below the version that defines it is refused", func(t *testing.T) {
+			for _, version := range contractVersions {
+				if version >= ebsprovider.NoSealSchemaVersion {
+					continue
+				}
+				t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) {
+					provider := newProvider(t)
+					ctx := context.Background()
+					volumeID := cfg.id(fmt.Sprintf("vol-unpub-noseal-v%d", version))
+					_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+					require.NoError(t, err)
+					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+					require.NoError(t, err)
+
+					err = unpublish(ctx, provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: volumeID, NodeID: cfg.node(), NoSeal: true})
+					require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument,
+						"a version %d server would ignore no_seal and seal, so the flag must be refused rather than dropped", version)
+					err = provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID})
+					require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse, "the refused unpublish must have left the volume published")
+				})
+			}
+		})
+
+		t.Run("no_seal is honoured or refused as unsupported", func(t *testing.T) {
+			provider := newProvider(t)
+			ctx := context.Background()
+			volumeID := cfg.id("vol-unpub-noseal")
+			v3 := ebsprovider.Versioned{SchemaVersion: ebsprovider.NoSealSchemaVersion}
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+
+			result, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: v3, VolumeID: volumeID, NodeID: cfg.node(), NoSeal: true})
+			if errors.Is(err, ebsprovider.ErrUnsupportedVersion) {
+				return
+			}
+			require.NoError(t, err)
+			assert.True(t, result.TornDown, "the volume was published, so a no_seal unpublish removed an export")
+			again, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: v3, VolumeID: volumeID, NodeID: cfg.node(), NoSeal: true})
+			require.NoError(t, err)
+			assert.False(t, again.TornDown)
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err, "a volume dropped without sealing must still be publishable")
 		})
 
 		t.Run("invalid_argument on missing ids", func(t *testing.T) {
 			provider := newProvider(t)
-			err := provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned()})
+			err := unpublish(context.Background(), provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned()})
 			require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
 		})
 	})
 
+	t.Run("HoldsUnsealedWrites", func(t *testing.T) {
+		t.Run("answers with a subset of the volumes asked about", func(t *testing.T) {
+			provider := newProvider(t)
+			ctx := context.Background()
+			volumeID := cfg.id("vol-unsealed")
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+
+			asked := []string{volumeID, cfg.id("vol-unsealed-never")}
+			held, err := provider.HoldsUnsealedWrites(ctx, cfg.node(), asked)
+			require.NoError(t, err, "an error reads as cannot say, which blocks every start elsewhere")
+			assert.Subset(t, asked, held, "a volume nobody asked about is not an answer to the question")
+		})
+	})
+
 	t.Run("Exclusion", func(t *testing.T) {
 		exclusion := capabilities.Exclusion
 
@@ -907,7 +1117,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 
 			// Releasing the claim must make the volume publishable again, or
 			// the guarantee is indistinguishable from a permanent lock.
-			require.NoError(t, provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()}))
+			require.NoError(t, unpublish(ctx, provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()}))
 			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.otherNode()})
 			require.NoError(t, err, "a released claim must be reclaimable, or an unpublish strands the volume")
 		})
@@ -968,7 +1178,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse,
 				"deleting a published volume must be refused; the export is still serving reads and writes against it")
 
-			require.NoError(t, provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()}))
+			require.NoError(t, unpublish(ctx, provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()}))
 			require.NoError(t, provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID}),
 				"once unpublished the delete must go through, or a volume can never be removed after being used")
 		})

spinifex/providers/ebs/conformance/nbdclient.go

@@ -217,7 +217,7 @@ func cleanupVolume(t *testing.T, provider ebsprovider.EBSProvider, cfg NBDClient
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
-	if err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
+	if _, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.NodeID,
 	}); err != nil {
 		t.Logf("cleanup: unpublish %s: %v", volumeID, err)
@@ -310,7 +310,7 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
 	// holes to a server that really tracks allocation.
 	t.Run("sparse extent reporting matches the advertised capability", func(t *testing.T) {
 		provider := newProvider(t)
-		capabilities := capabilitiesOf(t, provider)
+		capabilities := capabilitiesOf(t, provider).Capabilities
 		nbdURI := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-extents", false)
 
 		holes := slices.ContainsFunc(nbdMap(t, dialURI(t, nbdURI)), func(extent nbdExtent) bool {
@@ -323,7 +323,7 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
 
 	t.Run("read-only publish matches the advertised capability", func(t *testing.T) {
 		provider := newProvider(t)
-		capabilities := capabilitiesOf(t, provider)
+		capabilities := capabilitiesOf(t, provider).Capabilities
 
 		if !capabilities.ReadOnlyPublish {
 			// Refusing is the contract for a provider that cannot honour it;
@@ -356,11 +356,12 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
 		uri := dialURI(t, nbdURI)
 		require.True(t, nbdReachable(uri), "export should be reachable before unpublish")
 
-		require.NoError(t, provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{
+		_, err := provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{
 			Versioned: ebsprovider.NewVersioned(),
 			VolumeID:  volumeID,
 			NodeID:    cfg.NodeID,
-		}))
+		})
+		require.NoError(t, err)
 
 		assert.Eventually(t, func() bool { return !nbdReachable(uri) }, 30*time.Second, 250*time.Millisecond,
 			"export still answers after UnpublishVolume")

spinifex/providers/ebs/conformance/tracking.go

@@ -84,12 +84,12 @@ func (p *trackingProvider) PublishVolume(ctx context.Context, req ebsprovider.Pu
 	return pub, err
 }
 
-func (p *trackingProvider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error {
-	err := p.EBSProvider.UnpublishVolume(ctx, req)
+func (p *trackingProvider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
+	result, err := p.EBSProvider.UnpublishVolume(ctx, req)
 	if err == nil {
 		delete(p.published, req.VolumeID)
 	}
-	return err
+	return result, err
 }
 
 // cleanup removes what the subtest left behind, in dependency order: a
@@ -100,7 +100,7 @@ func (p *trackingProvider) cleanup() {
 	ctx := context.Background()
 
 	for volumeID, nodeID := range p.published {
-		if err := p.EBSProvider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
+		if _, err := p.EBSProvider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
 			Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, Handle: p.volumes[volumeID], NodeID: nodeID,
 		}); err != nil {
 			p.t.Logf("conformance cleanup: unpublish %s: %v", volumeID, err)

spinifex/providers/ebs/errors.go

@@ -12,6 +12,11 @@ var (
 	ErrUnsupportedVersion = errors.New("unsupported EBS provider schema version")
 	ErrVolumeInUse        = errors.New("volume is in use")
 
+	// ErrVolumeHeldElsewhere is a publish refused because another node holds
+	// the volume's claim. It wraps ErrVolumeInUse, so a caller that only asks
+	// whether the volume is in use still matches it.
+	ErrVolumeHeldElsewhere = fmt.Errorf("volume is held by another node: %w", ErrVolumeInUse)
+
 	// ErrUnsupportedCapability is returned when a request asks for optional
 	// behaviour this provider does not advertise in Capabilities. It is
 	// distinct from ErrInvalidArgument: the request is well formed.
@@ -28,19 +33,35 @@ var (
 	ErrUnavailable = errors.New("provider is temporarily unable to answer")
 )
 
+// HeldElsewhereSchemaVersion is the first version a held-elsewhere refusal is
+// reported as such. Earlier requests get volume_in_use, as v1.23.0 sends it.
+const HeldElsewhereSchemaVersion uint16 = 3
+
+// HeldElsewhereError is the sentinel a held-elsewhere refusal of a request at
+// version wraps: ErrVolumeHeldElsewhere from HeldElsewhereSchemaVersion on.
+func HeldElsewhereError(version uint16) error {
+	if version >= HeldElsewhereSchemaVersion {
+		return ErrVolumeHeldElsewhere
+	}
+	return ErrVolumeInUse
+}
+
 // ErrorCode is the wire form of a provider error in ProviderError. Unwrap maps
 // every code but internal back to the matching Err sentinel.
 type ErrorCode string
 
 const (
-	ErrorCodeAlreadyExists      ErrorCode = "already_exists"
-	ErrorCodeInvalidArgument    ErrorCode = "invalid_argument"
-	ErrorCodeNotFound           ErrorCode = "not_found"
-	ErrorCodeUnsupportedVersion ErrorCode = "unsupported_version"
-	ErrorCodeVolumeInUse        ErrorCode = "volume_in_use"
-	ErrorCodeUnsupportedCap     ErrorCode = "unsupported_capability"
-	ErrorCodeUnavailable        ErrorCode = "unavailable"
-	ErrorCodeInternal           ErrorCode = "internal"
+	ErrorCodeAlreadyExists       ErrorCode = "already_exists"
+	ErrorCodeInvalidArgument     ErrorCode = "invalid_argument"
+	ErrorCodeNotFound            ErrorCode = "not_found"
+	ErrorCodeUnsupportedVersion  ErrorCode = "unsupported_version"
+	ErrorCodeVolumeInUse         ErrorCode = "volume_in_use"
+	ErrorCodeVolumeHeldElsewhere ErrorCode = "volume_held_elsewhere"
+	ErrorCodeUnsupportedCap      ErrorCode = "unsupported_capability"
+	// ErrorCodeUnavailable is the retry-later code: among others, a mount
+	// against a backing store that is not ready yet returns it.
+	ErrorCodeUnavailable ErrorCode = "unavailable"
+	ErrorCodeInternal    ErrorCode = "internal"
 )
 
 // Retryable reports whether repeating the identical request could succeed.
@@ -78,6 +99,8 @@ func (e *ProviderError) Unwrap() error {
 		return ErrUnsupportedVersion
 	case ErrorCodeVolumeInUse:
 		return ErrVolumeInUse
+	case ErrorCodeVolumeHeldElsewhere:
+		return ErrVolumeHeldElsewhere
 	case ErrorCodeUnsupportedCap:
 		return ErrUnsupportedCapability
 	case ErrorCodeUnavailable:
@@ -89,7 +112,12 @@ func (e *ProviderError) Unwrap() error {
 
 func checkVersion(version uint16) error {
 	if !SupportsSchemaVersion(version) {
-		return fmt.Errorf("%w: got %d, supported versions are %d and %d", ErrUnsupportedVersion, version, LegacySchemaVersion, SchemaVersion)
+		return fmt.Errorf("%w: got %d, %s", ErrUnsupportedVersion, version, SupportedVersionsText())
 	}
 	return nil
 }
+
+// SupportedVersionsText names the served version range for error messages.
+func SupportedVersionsText() string {
+	return fmt.Sprintf("supported versions are %d to %d", LegacySchemaVersion, SchemaVersion)
+}

spinifex/providers/ebs/hostile/soak_test.go

@@ -105,7 +105,7 @@ func soakOneVolume(ctx context.Context, provider ebsprovider.EBSProvider, volume
 	if _, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: "node-1",
 	}); err == nil {
-		_ = provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
+		_, _ = provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
 			Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: "node-1",
 		})
 	}

spinifex/providers/ebs/hostile/verbs.go

@@ -21,6 +21,7 @@ const (
 	verbSnapshotDelete  = "snapshot.delete"
 	verbSnapshotCopy    = "snapshot.copy"
 	verbSnapshotList    = "snapshot.list"
+	verbUnsealedWrites  = "volume.unsealed"
 )
 
 // lieFlavours are the ways a successful answer can be wrong. Each one is a
@@ -240,10 +241,28 @@ func (p *Provider) PublishVolume(ctx context.Context, req ebsprovider.PublishVol
 	return published, nil
 }
 
-func (p *Provider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error {
+func (p *Provider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
 	injection := p.draw(verbVolumeUnpublish, req.VolumeID)
 	if stop, err := p.apply(ctx, injection); stop || err != nil {
-		return err
+		return nil, err
 	}
-	return p.after(injection, p.inner.UnpublishVolume(ctx, req))
+	result, err := p.inner.UnpublishVolume(ctx, req)
+	if err := p.after(injection, err); err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+// HoldsUnsealedWrites is never made to lie: a false "none held" lets an
+// instance start elsewhere over writes only this node has.
+func (p *Provider) HoldsUnsealedWrites(ctx context.Context, nodeID string, volumeIDs []string) ([]string, error) {
+	injection := p.drawTruthful(verbUnsealedWrites, nodeID)
+	if stop, err := p.apply(ctx, injection); stop || err != nil {
+		return nil, err
+	}
+	held, err := p.inner.HoldsUnsealedWrites(ctx, nodeID, volumeIDs)
+	if err := p.after(injection, err); err != nil {
+		return nil, err
+	}
+	return held, nil
 }

spinifex/providers/ebs/memory.go

@@ -27,6 +27,7 @@ type memoryVolume struct {
 	parameters       []byte
 	seed             []byte
 	published        *PublishedVolume
+	readOnly         bool
 }
 
 var _ EBSProvider = (*MemoryProvider)(nil)
@@ -53,7 +54,11 @@ func (m *MemoryProvider) GetCapabilities(_ context.Context, req GetCapabilitiesR
 	if err := checkVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
-	return &GetCapabilitiesResponse{Versioned: NewVersioned(), Capabilities: m.capabilities}, nil
+	return &GetCapabilitiesResponse{
+		Versioned:        ReplyVersioned(req.SchemaVersion),
+		Capabilities:     m.capabilities,
+		ContractVersions: SupportedSchemaVersions(),
+	}, nil
 }
 
 func (m *MemoryProvider) CreateVolume(_ context.Context, req CreateVolumeRequest) (*Volume, error) {
@@ -167,7 +172,7 @@ func (m *MemoryProvider) ListVolumes(_ context.Context, req ListVolumesRequest)
 	defer m.mu.RUnlock()
 	ids := slices.Sorted(maps.Keys(m.volumes))
 
-	response := &ListVolumesResponse{Versioned: NewVersioned()}
+	response := &ListVolumesResponse{Versioned: ReplyVersioned(req.SchemaVersion)}
 	response.Volumes, response.NextToken = Page(ids, req.StartingToken, int(req.PageSize()),
 		func(id string) VolumeRef {
 			return VolumeRef{ID: id, Handle: m.volumes[id].volume.Handle}
@@ -338,7 +343,7 @@ func (m *MemoryProvider) ListSnapshots(_ context.Context, req ListSnapshotsReque
 	defer m.mu.RUnlock()
 	ids := slices.Sorted(maps.Keys(m.snapshots))
 
-	response := &ListSnapshotsResponse{Versioned: NewVersioned()}
+	response := &ListSnapshotsResponse{Versioned: ReplyVersioned(req.SchemaVersion)}
 	response.Snapshots, response.NextToken = Page(ids, req.StartingToken, int(req.PageSize()),
 		func(id string) SnapshotRef {
 			return SnapshotRef{
@@ -368,7 +373,10 @@ func (m *MemoryProvider) PublishVolume(_ context.Context, req PublishVolumeReque
 	}
 	if volume.published != nil {
 		if volume.published.NodeID != req.NodeID {
-			return nil, fmt.Errorf("%w: volume %s is published to %s", ErrVolumeInUse, req.VolumeID, volume.published.NodeID)
+			return nil, fmt.Errorf("%w: volume %s is published to %s", HeldElsewhereError(req.SchemaVersion), req.VolumeID, volume.published.NodeID)
+		}
+		if volume.readOnly != req.ReadOnly {
+			return nil, fmt.Errorf("%w: volume %s is already published read_only=%t", ErrVolumeInUse, req.VolumeID, volume.readOnly)
 		}
 		return clonePublished(volume.published), nil
 	}
@@ -377,26 +385,44 @@ func (m *MemoryProvider) PublishVolume(_ context.Context, req PublishVolumeReque
 		NodeID:   req.NodeID,
 		Export:   NewNBDExport("nbd+unix:///?socket=/memory/" + req.VolumeID + ".sock"),
 	}
+	volume.readOnly = req.ReadOnly
 	volume.volume.State = VolumeStateInUse
 	return clonePublished(volume.published), nil
 }
 
-func (m *MemoryProvider) UnpublishVolume(_ context.Context, req UnpublishVolumeRequest) error {
-	if err := checkVersion(req.SchemaVersion); err != nil {
-		return err
+// UnpublishVolume honours NoSeal trivially: the volume's bytes live in this
+// process, so there is no local copy to seal or to drop.
+func (m *MemoryProvider) UnpublishVolume(_ context.Context, req UnpublishVolumeRequest) (*UnpublishedVolume, error) {
+	if err := CheckUnpublishVersion(req); err != nil {
+		return nil, err
 	}
 	if req.VolumeID == "" || req.NodeID == "" {
-		return fmt.Errorf("%w: volume and node IDs are required", ErrInvalidArgument)
+		return nil, fmt.Errorf("%w: volume and node IDs are required", ErrInvalidArgument)
 	}
 	m.mu.Lock()
 	defer m.mu.Unlock()
+	result := &UnpublishedVolume{}
 	volume := m.volumes[req.VolumeID]
 	if volume == nil || volume.published == nil || volume.published.NodeID != req.NodeID {
-		return nil
+		return result, nil
 	}
 	volume.published = nil
+	volume.readOnly = false
 	volume.volume.State = VolumeStateAvailable
-	return nil
+	result.TornDown = req.SchemaVersion >= NoSealSchemaVersion
+	return result, nil
+}
+
+// HoldsUnsealedWrites answers that none are held: every write lands in this
+// process's map, which is the backend, before it is acknowledged.
+func (m *MemoryProvider) HoldsUnsealedWrites(_ context.Context, nodeID string, volumeIDs []string) ([]string, error) {
+	if nodeID == "" {
+		return nil, fmt.Errorf("%w: node ID is required", ErrInvalidArgument)
+	}
+	if slices.Contains(volumeIDs, "") {
+		return nil, fmt.Errorf("%w: empty volume ID", ErrInvalidArgument)
+	}
+	return nil, nil
 }
 
 func cloneVolume(volume *Volume) *Volume {

spinifex/providers/ebs/memory_test.go

@@ -56,9 +56,10 @@ func TestMemoryProviderLifecycleAndIdempotency(t *testing.T) {
 		Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle,
 	}), ErrVolumeInUse)
 
-	require.NoError(t, provider.UnpublishVolume(ctx, UnpublishVolumeRequest{
+	_, err = provider.UnpublishVolume(ctx, UnpublishVolumeRequest{
 		Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle, NodeID: "node-1",
-	}))
+	})
+	require.NoError(t, err)
 
 	expanded, err := provider.ExpandVolume(ctx, ExpandVolumeRequest{
 		Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle,
@@ -141,7 +142,7 @@ func TestMemoryProviderChecksVersionOnEveryMethod(t *testing.T) {
 	_, err = provider.PublishVolume(ctx, PublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"})
 	require.ErrorIs(t, err, ErrUnsupportedVersion)
 
-	err = provider.UnpublishVolume(ctx, UnpublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"})
+	_, err = provider.UnpublishVolume(ctx, UnpublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"})
 	require.ErrorIs(t, err, ErrUnsupportedVersion)
 }
 

spinifex/providers/ebs/nats.go

@@ -39,20 +39,22 @@ const defaultRequestTimeout = 30 * time.Second
 // codebase's existing budget for an EBS operation the caller cannot bound.
 const dataMovementTimeout = 180 * time.Second
 
-// dataMovementSubjects names every subject that references or moves real data.
-// Everything absent answers from state already held and keeps the caller's
-// timeout, so a subject added without a deliberate decision stays fast.
-var dataMovementSubjects = map[string]struct{}{
-	CreateVolumeSubject:   {},
-	ExpandVolumeSubject:   {},
-	CopySnapshotSubject:   {},
-	DeleteVolumeSubject:   {},
-	DeleteSnapshotSubject: {},
+// dataMovementVerbs names every operation that references or moves real data,
+// however its subject addresses it. Everything absent answers from state
+// already held and keeps the caller's timeout, so a new verb stays fast.
+var dataMovementVerbs = map[string]bool{
+	verbVolumeCreate:   true,
+	verbVolumeExpand:   true,
+	verbVolumeDelete:   true,
+	verbSnapshotCreate: true,
+	verbSnapshotCopy:   true,
+	verbSnapshotDelete: true,
+	// Unpublish seals local state back to the backend, which takes as long as
+	// the volume has unsealed writes.
+	verbVolumeUnpublish: true,
 }
 
-// timeoutFor sizes a request to the work its subject names. The per-volume
-// snapshot create and the owner-routed variants carry no constant, so they are
-// matched by prefix and by verb.
+// timeoutFor sizes a request to the work its subject names.
 func (p *NATSProvider) timeoutFor(subject string) time.Duration {
 	if movesData(subject) {
 		return max(p.requestTimeout, dataMovementTimeout)
@@ -60,28 +62,8 @@ func (p *NATSProvider) timeoutFor(subject string) time.Duration {
 	return p.requestTimeout
 }
 
-// movesData answers for the named subjects, for the per-volume snapshot create
-// prefix, and for the owner-routed variants, none of which carry a constant.
 func movesData(subject string) bool {
-	if _, ok := dataMovementSubjects[subject]; ok {
-		return true
-	}
-	if strings.HasPrefix(subject, SnapshotCreateSubjectPrefix) {
-		return true
-	}
-	_, verb, ok := ParseOwnerSubject(subject)
-	return ok && isDataMovementVerb(verb)
-}
-
-// isDataMovementVerb answers for the owner-routed subjects. volume.describe is
-// the one owner verb that reads state rather than moving data.
-func isDataMovementVerb(verb string) bool {
-	switch verb {
-	case verbSnapshotCreate, verbSnapshotCopy, verbVolumeExpand:
-		return true
-	default:
-		return false
-	}
+	return dataMovementVerbs[parseSubject(subject).verb]
 }
 
 // SnapshotSubject addresses a create by source volume so the owning node can
@@ -104,32 +86,65 @@ func SnapshotCompletionSubject(snapshotID string) (string, error) {
 	return "ebs.provider.v1.snapshot.response." + snapshotID, nil
 }
 
-// PublishSubject returns nodeID's per-node mount subject
-// (ebs.provider.v1.<node>.mount), which serves PublishVolume on that node.
-func PublishSubject(nodeID string) (string, error) {
+// Last tokens of the per-node subjects: ebs.provider.v1.<node>.<verb>.
+const (
+	publishVerb          = "mount"
+	unpublishVerb        = "unmount"
+	unsealedVerb         = "unsealed"
+	nodeCapabilitiesVerb = "capabilities"
+	fenceNoticeVerb      = "fenced"
+)
+
+// nodeSubjectWildcard is the subject matching verb on every node, for a server
+// that stands in for every node's provider.
+func nodeSubjectWildcard(verb string) string { return subjectPrefix + "*." + verb }
+
+// Wildcards over every node for each per-node subject.
+var (
+	PublishSubjectWildcard          = nodeSubjectWildcard(publishVerb)
+	UnpublishSubjectWildcard        = nodeSubjectWildcard(unpublishVerb)
+	UnsealedWritesSubjectWildcard   = nodeSubjectWildcard(unsealedVerb)
+	NodeCapabilitiesSubjectWildcard = nodeSubjectWildcard(nodeCapabilitiesVerb)
+)
+
+func nodeSubject(nodeID, verb string) (string, error) {
 	if err := validateSubjectToken(nodeID); err != nil {
 		return "", err
 	}
-	return subjectPrefix + nodeID + ".mount", nil
+	return subjectPrefix + nodeID + "." + verb, nil
+}
+
+// PublishSubject returns nodeID's per-node mount subject
+// (ebs.provider.v1.<node>.mount), which serves PublishVolume on that node.
+func PublishSubject(nodeID string) (string, error) {
+	return nodeSubject(nodeID, publishVerb)
 }
 
 // UnpublishSubject returns nodeID's per-node unmount subject
 // (ebs.provider.v1.<node>.unmount), which serves UnpublishVolume on that node.
 func UnpublishSubject(nodeID string) (string, error) {
-	if err := validateSubjectToken(nodeID); err != nil {
-		return "", err
-	}
-	return subjectPrefix + nodeID + ".unmount", nil
+	return nodeSubject(nodeID, unpublishVerb)
 }
 
 // UnsealedWritesSubject returns nodeID's per-node subject
 // (ebs.provider.v1.<node>.unsealed), which answers whether that node holds
 // unsealed writes for a set of volumes.
 func UnsealedWritesSubject(nodeID string) (string, error) {
-	if err := validateSubjectToken(nodeID); err != nil {
-		return "", err
-	}
-	return subjectPrefix + nodeID + ".unsealed", nil
+	return nodeSubject(nodeID, unsealedVerb)
+}
+
+// NodeCapabilitiesSubject returns nodeID's per-node GetCapabilities subject
+// (ebs.provider.v1.<node>.capabilities). Unlike CapabilitiesSubject, which
+// any node's provider answers, only nodeID's provider answers it.
+func NodeCapabilitiesSubject(nodeID string) (string, error) {
+	return nodeSubject(nodeID, nodeCapabilitiesVerb)
+}
+
+// FenceNoticeSubject returns nodeID's fence-notice subject
+// (ebs.provider.v1.<node>.fenced), on which a provider that revoked a live
+// export there sends a FenceNotice for that node's daemon to acknowledge.
+func FenceNoticeSubject(nodeID string) (string, error) {
+	return nodeSubject(nodeID, fenceNoticeVerb)
 }
 
 // ownerSubjectPrefix is common to every owner-routed subject OwnerSubject
@@ -224,11 +239,26 @@ func NewNATSProvider(conn *nats.Conn, requestTimeout time.Duration) *NATSProvide
 }
 
 func (p *NATSProvider) GetCapabilities(ctx context.Context, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
+	return p.getCapabilities(ctx, CapabilitiesSubject, req)
+}
+
+// GetNodeCapabilities asks nodeID's own provider what it supports, over
+// NodeCapabilitiesSubject. A provider that predates that subject gets no
+// responder, which a caller must count as no evidence.
+func (p *NATSProvider) GetNodeCapabilities(ctx context.Context, nodeID string, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
+	subject, err := NodeCapabilitiesSubject(nodeID)
+	if err != nil {
+		return nil, err
+	}
+	return p.getCapabilities(ctx, subject, req)
+}
+
+func (p *NATSProvider) getCapabilities(ctx context.Context, subject string, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
 	if err := checkVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	var response GetCapabilitiesResponse
-	if err := p.request(ctx, CapabilitiesSubject, req, &response); err != nil {
+	if err := p.request(ctx, subject, req, &response); err != nil {
 		return nil, err
 	}
 	if err := responseError(response.SchemaVersion, response.Error); err != nil {
@@ -490,19 +520,22 @@ func (p *NATSProvider) PublishVolume(ctx context.Context, req PublishVolumeReque
 	return response.Published, nil
 }
 
-func (p *NATSProvider) UnpublishVolume(ctx context.Context, req UnpublishVolumeRequest) error {
-	if err := checkVersion(req.SchemaVersion); err != nil {
-		return err
+func (p *NATSProvider) UnpublishVolume(ctx context.Context, req UnpublishVolumeRequest) (*UnpublishedVolume, error) {
+	if err := CheckUnpublishVersion(req); err != nil {
+		return nil, err
 	}
 	subject, err := UnpublishSubject(req.NodeID)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	var response UnpublishVolumeResponse
 	if err := p.request(ctx, subject, req, &response); err != nil {
-		return err
+		return nil, err
 	}
-	return responseError(response.SchemaVersion, response.Error)
+	if err := responseError(response.SchemaVersion, response.Error); err != nil {
+		return nil, err
+	}
+	return &UnpublishedVolume{TornDown: response.TornDown}, nil
 }
 
 // HoldsUnsealedWrites returns the subset of volumeIDs for which nodeID's

spinifex/providers/ebs/nats_test.go

@@ -250,7 +250,7 @@ func TestNATSProviderChecksVersionBeforeTransport(t *testing.T) {
 		require.ErrorIs(t, err, ErrUnsupportedVersion)
 	})
 	t.Run("UnpublishVolume", func(t *testing.T) {
-		err := provider.UnpublishVolume(ctx, UnpublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"})
+		_, err := provider.UnpublishVolume(ctx, UnpublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"})
 		require.ErrorIs(t, err, ErrUnsupportedVersion)
 	})
 }
@@ -563,7 +563,7 @@ func TestNATSProviderPropagatesProviderErrors(t *testing.T) {
 		respondWith(t, conn, subject, UnpublishVolumeResponse{
 			Versioned: NewVersioned(), Error: &ProviderError{Code: ErrorCodeNotFound, Message: "no such mount"},
 		})
-		err = NewNATSProvider(conn, time.Second).UnpublishVolume(t.Context(), UnpublishVolumeRequest{
+		_, err = NewNATSProvider(conn, time.Second).UnpublishVolume(t.Context(), UnpublishVolumeRequest{
 			Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-err",
 		})
 		require.ErrorIs(t, err, ErrNotFound)
@@ -601,7 +601,7 @@ func TestNATSProviderValidatesSubjectTokensBeforeTransport(t *testing.T) {
 		require.ErrorIs(t, err, ErrInvalidArgument)
 	})
 	t.Run("UnpublishVolume", func(t *testing.T) {
-		err := provider.UnpublishVolume(ctx, UnpublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node.*"})
+		_, err := provider.UnpublishVolume(ctx, UnpublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node.*"})
 		require.ErrorIs(t, err, ErrInvalidArgument)
 	})
 }
@@ -712,6 +712,14 @@ func TestTimeoutForSizesEachSubjectToItsWork(t *testing.T) {
 	require.NoError(t, err)
 	ownerDescribe, err := GetVolumeOwnerSubject("vol-1")
 	require.NoError(t, err)
+	unpublish, err := UnpublishSubject("node-1")
+	require.NoError(t, err)
+	publish, err := PublishSubject("node-1")
+	require.NoError(t, err)
+	unsealed, err := UnsealedWritesSubject("node-1")
+	require.NoError(t, err)
+	nodeCapabilities, err := NodeCapabilitiesSubject("node-1")
+	require.NoError(t, err)
 
 	// A delete costs one round trip per object, so its duration is a function
 	// of the volume. On prod a 100 GiB volume's delete took 52 s against a 30 s
@@ -721,6 +729,8 @@ func TestTimeoutForSizesEachSubjectToItsWork(t *testing.T) {
 		CreateVolumeSubject, ExpandVolumeSubject, CopySnapshotSubject,
 		DeleteVolumeSubject, DeleteSnapshotSubject,
 		snapshotCreate, ownerCopy, ownerExpand, ownerCreate,
+		// Unpublish seals the volume's unsealed writes back to the backend.
+		unpublish,
 	}
 	for _, subject := range slow {
 		assert.Equal(t, dataMovementTimeout, p.timeoutFor(subject),
@@ -730,6 +740,7 @@ func TestTimeoutForSizesEachSubjectToItsWork(t *testing.T) {
 	fast := []string{
 		CapabilitiesSubject, GetVolumeSubject, ListVolumesSubject,
 		ListSnapshotsSubject, ownerDescribe,
+		publish, unsealed, nodeCapabilities,
 	}
 	for _, subject := range fast {
 		assert.Equal(t, callerTimeout, p.timeoutFor(subject),

spinifex/providers/ebs/natsserve/classify_test.go

@@ -23,6 +23,7 @@ func TestClassifyError(t *testing.T) {
 		{ebsprovider.ErrVolumeInUse, ebsprovider.ErrorCodeVolumeInUse},
 		{ebsprovider.ErrUnsupportedCapability, ebsprovider.ErrorCodeUnsupportedCap},
 		{ebsprovider.ErrUnavailable, ebsprovider.ErrorCodeUnavailable},
+		{ebsprovider.ErrVolumeHeldElsewhere, ebsprovider.ErrorCodeVolumeHeldElsewhere},
 	}
 
 	for _, tt := range tests {

spinifex/providers/ebs/natsserve/handlers_test.go

@@ -96,8 +96,12 @@ func (p *failingProvider) PublishVolume(context.Context, ebsprovider.PublishVolu
 	return nil, p.err
 }
 
-func (p *failingProvider) UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) error {
-	return p.err
+func (p *failingProvider) UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
+	return nil, p.err
+}
+
+func (p *failingProvider) HoldsUnsealedWrites(context.Context, string, []string) ([]string, error) {
+	return nil, p.err
 }
 
 // wireVerb names one subject Serve subscribes, a valid request body for it,
@@ -120,6 +124,10 @@ func wireVerbs(t *testing.T) []wireVerb {
 	require.NoError(t, err)
 	unmountSubject, err := ebsprovider.UnpublishSubject("node-1")
 	require.NoError(t, err)
+	unsealedSubject, err := ebsprovider.UnsealedWritesSubject("node-1")
+	require.NoError(t, err)
+	nodeCapabilitiesSubject, err := ebsprovider.NodeCapabilitiesSubject("node-1")
+	require.NoError(t, err)
 
 	return []wireVerb{
 		{name: "capabilities", subject: ebsprovider.CapabilitiesSubject,
@@ -146,9 +154,122 @@ func wireVerbs(t *testing.T) []wireVerb {
 			request: ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", NodeID: "node-1"}},
 		{name: "unmount", subject: unmountSubject,
 			request: ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", NodeID: "node-1"}},
+		{name: "unsealed", subject: unsealedSubject,
+			request: ebsprovider.HoldsUnsealedWritesRequest{Versioned: ebsprovider.NewVersioned(), NodeID: "node-1", VolumeIDs: []string{"vol-1"}}},
+		{name: "node.capabilities", subject: nodeCapabilitiesSubject,
+			request: ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}},
 	}
 }
 
+// withVersion re-encodes a wireVerb request at version, so one table drives
+// every served version without a copy of each request per version.
+func withVersion(t *testing.T, request any, version uint16) []byte {
+	t.Helper()
+	payload, err := json.Marshal(request)
+	require.NoError(t, err)
+	var body map[string]any
+	require.NoError(t, json.Unmarshal(payload, &body))
+	body["schema_version"] = version
+	payload, err = json.Marshal(body)
+	require.NoError(t, err)
+	return payload
+}
+
+// TestHandlersReplyInRequestVersion is what lets a version 3 server serve a
+// v1.23.0 caller: every reply, success or provider failure, carries the
+// version the request was sent at, never the server's newest.
+func TestHandlersReplyInRequestVersion(t *testing.T) {
+	failing := serveProvider(t, &failingProvider{err: fmt.Errorf("vol-1: %w", ebsprovider.ErrUnavailable)}, natsserve.Options{NoQueueGroup: true})
+	for _, version := range ebsprovider.SupportedSchemaVersions() {
+		for _, verb := range wireVerbs(t) {
+			// Each success case gets its own seeded provider, so none relies on
+			// an earlier case having created or deleted something.
+			t.Run(fmt.Sprintf("success/v%d/%s", version, verb.name), func(t *testing.T) {
+				conn := serveProvider(t, seededProvider(t), natsserve.Options{NoQueueGroup: true})
+				for _, envelope := range requestAt(t, conn, verb, version) {
+					require.Nil(t, envelope.Error, "the case must succeed for it to check a success reply")
+					assert.Equal(t, version, envelope.SchemaVersion)
+				}
+			})
+			t.Run(fmt.Sprintf("failure/v%d/%s", version, verb.name), func(t *testing.T) {
+				envelopes := requestAt(t, failing, verb, version)
+				// A snapshot create is accepted first; the failure is in its completion.
+				require.NotNil(t, envelopes[len(envelopes)-1].Error, "the case must fail for it to check a failure reply")
+				for _, envelope := range envelopes {
+					assert.Equal(t, version, envelope.SchemaVersion)
+				}
+			})
+		}
+	}
+}
+
+// seededProvider holds what every wireVerbs request needs to succeed: vol-1
+// and snap-1 taken from it, with enumeration advertised so listing succeeds.
+func seededProvider(t *testing.T) ebsprovider.EBSProvider {
+	t.Helper()
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeEnumeration: true, SnapshotEnumeration: true})
+	ctx := context.Background()
+	_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+	require.NoError(t, err)
+	_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: "snap-1", VolumeID: "vol-1"})
+	require.NoError(t, err)
+	return provider
+}
+
+// requestAt sends verb at version and returns the reply envelope, followed by
+// the completion's for a verb that publishes one.
+func requestAt(t *testing.T, conn *nats.Conn, verb wireVerb, version uint16) []wireEnvelope {
+	t.Helper()
+	var completionSub *nats.Subscription
+	if verb.completionSubject != "" {
+		var err error
+		completionSub, err = conn.SubscribeSync(verb.completionSubject)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = completionSub.Unsubscribe() })
+		require.NoError(t, conn.Flush())
+	}
+
+	msg, err := conn.Request(verb.subject, withVersion(t, verb.request, version), requestTimeout)
+	require.NoError(t, err)
+	envelopes := []wireEnvelope{decodeEnvelope(t, msg.Data)}
+	if completionSub != nil {
+		msg, err = completionSub.NextMsg(requestTimeout)
+		require.NoError(t, err)
+		envelopes = append(envelopes, decodeEnvelope(t, msg.Data))
+	}
+	return envelopes
+}
+
+// TestUnpublishNoSealOnTheWire pins the server half of no_seal: refused below
+// version 3 without reaching the provider, and reported as torn down at 3.
+func TestUnpublishNoSealOnTheWire(t *testing.T) {
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
+	conn := serveProvider(t, provider, natsserve.Options{NoQueueGroup: true})
+	ctx := context.Background()
+	_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", NodeID: "node-1"})
+	require.NoError(t, err)
+	subject, err := ebsprovider.UnpublishSubject("node-1")
+	require.NoError(t, err)
+	request := ebsprovider.UnpublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1", NoSeal: true}
+
+	msg, err := conn.Request(subject, withVersion(t, request, ebsprovider.DescriptorSchemaVersion), requestTimeout)
+	require.NoError(t, err)
+	envelope := decodeEnvelope(t, msg.Data)
+	require.NotNil(t, envelope.Error)
+	assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, envelope.Error.Code)
+	err = provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1"})
+	require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse, "the refused unpublish must not have reached the provider")
+
+	msg, err = conn.Request(subject, withVersion(t, request, ebsprovider.NoSealSchemaVersion), requestTimeout)
+	require.NoError(t, err)
+	var resp ebsprovider.UnpublishVolumeResponse
+	require.NoError(t, json.Unmarshal(msg.Data, &resp))
+	require.Nil(t, resp.Error)
+	assert.True(t, resp.TornDown)
+}
+
 // TestHandlersRejectMalformedRequests pins the caller-fault arm of every
 // subject: a body that is not JSON must come back as a versioned
 // invalid_argument reply, not as a dropped message the caller waits out.
@@ -184,7 +305,8 @@ func TestHandlersRejectVersionSkew(t *testing.T) {
 			assert.Equal(t, ebsprovider.LegacySchemaVersion, envelope.SchemaVersion, "the refusal itself must be versioned")
 			require.NotNil(t, envelope.Error)
 			assert.Equal(t, ebsprovider.ErrorCodeUnsupportedVersion, envelope.Error.Code)
-			assert.Equal(t, fmt.Sprintf("unsupported schema version 99; supported versions are %d and %d", ebsprovider.LegacySchemaVersion, ebsprovider.SchemaVersion), envelope.Error.Message)
+			assert.Contains(t, envelope.Error.Message, "99", "the refusal must name the version it refused")
+			assert.Contains(t, envelope.Error.Message, ebsprovider.SupportedVersionsText(), "the refusal must name the versions served")
 		})
 	}
 }
@@ -282,9 +404,10 @@ func TestServedProviderRoundTrip(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, ebsprovider.VolumeStateInUse, inUse.State)
 
-	require.NoError(t, client.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
+	_, err = client.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-roundtrip", NodeID: "node-1",
-	}))
+	})
+	require.NoError(t, err)
 	available, err := client.GetVolume(ctx, ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-roundtrip"})
 	require.NoError(t, err)
 	assert.Equal(t, ebsprovider.VolumeStateAvailable, available.State)
@@ -614,6 +737,22 @@ func TestServeNodeIDScopesMountSubjects(t *testing.T) {
 	require.ErrorIs(t, err, nats.ErrNoResponders, "another node's mount subject must go unanswered")
 }
 
+// TestGetNodeCapabilitiesIsAnsweredOnlyByThatNode is why the per-node subject
+// exists: the shared one is answered by whichever node's provider is fastest,
+// which says nothing about the provider on the node a caller will mount on.
+func TestGetNodeCapabilitiesIsAnsweredOnlyByThatNode(t *testing.T) {
+	conn := serveProvider(t, ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}), natsserve.Options{NodeID: "node-1"})
+	client := ebsprovider.NewNATSProvider(conn, requestTimeout)
+	request := ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}
+
+	resp, err := client.GetNodeCapabilities(t.Context(), "node-1", request)
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.SupportedSchemaVersions(), resp.ContractVersions)
+
+	_, err = client.GetNodeCapabilities(t.Context(), "node-2", request)
+	require.ErrorIs(t, err, nats.ErrNoResponders, "another node's capabilities subject must go unanswered")
+}
+
 // TestServeRejectsUnroutableNodeID covers the subject-build failure path: an
 // unsubscribable node ID must fail Serve outright rather than leave a server
 // running with the shared subjects subscribed and no mount subject at all.
@@ -622,7 +761,7 @@ func TestServeRejectsUnroutableNodeID(t *testing.T) {
 
 	stop, err := natsserve.Serve(t.Context(), conn, ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}), natsserve.Options{NodeID: "node.*"})
 	require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
-	assert.Contains(t, err.Error(), "build publish subject")
+	assert.Contains(t, err.Error(), "build per-node subject for "+ebsprovider.PublishSubjectWildcard)
 	assert.Nil(t, stop)
 
 	_, err = conn.Request(ebsprovider.CapabilitiesSubject, []byte(fmt.Sprintf(`{"schema_version":%d}`, ebsprovider.SchemaVersion)), 200*time.Millisecond)

spinifex/providers/ebs/natsserve/server.go

@@ -36,9 +36,9 @@ type Options struct {
 	// across workers does not apply.
 	NoQueueGroup bool
 
-	// NodeID scopes PublishVolume/UnpublishVolume to this node's own
-	// subjects (ebsprovider.PublishSubject / UnpublishSubject), matching a
-	// production daemon that can only mount volumes on its own node. Left empty (the default), Serve instead subscribes the wildcard "ebs.provider.v1.*.mount"/"unmount" subjects, answering requests for any node.
+	// NodeID scopes the per-node subjects (publish, unpublish, unsealed writes
+	// and per-node capabilities) to this node, as a production daemon serves
+	// them. Left empty, Serve subscribes their wildcards and answers for any node.
 	NodeID string
 }
 
@@ -127,31 +127,27 @@ func Serve(ctx context.Context, nc *nats.Conn, provider ebsprovider.EBSProvider,
 		}
 	}
 
-	if opts.NodeID == "" {
-		if err := subscribePlain("ebs.provider.v1.*.mount", handlePublishVolume(provider)); err != nil {
-			cleanup()
-			return nil, err
+	nodeSteps := []struct {
+		wildcard string
+		build    func(string) (string, error)
+		handler  msgHandler
+	}{
+		{ebsprovider.PublishSubjectWildcard, ebsprovider.PublishSubject, handlePublishVolume(provider)},
+		{ebsprovider.UnpublishSubjectWildcard, ebsprovider.UnpublishSubject, handleUnpublishVolume(provider)},
+		{ebsprovider.UnsealedWritesSubjectWildcard, ebsprovider.UnsealedWritesSubject, handleHoldsUnsealedWrites(provider)},
+		{ebsprovider.NodeCapabilitiesSubjectWildcard, ebsprovider.NodeCapabilitiesSubject, handleCapabilities(provider)},
+	}
+	for _, step := range nodeSteps {
+		subject := step.wildcard
+		if opts.NodeID != "" {
+			built, err := step.build(opts.NodeID)
+			if err != nil {
+				cleanup()
+				return nil, fmt.Errorf("build per-node subject for %s: %w", step.wildcard, err)
+			}
+			subject = built
 		}
-		if err := subscribePlain("ebs.provider.v1.*.unmount", handleUnpublishVolume(provider)); err != nil {
-			cleanup()
-			return nil, err
-		}
-	} else {
-		publishSubject, err := ebsprovider.PublishSubject(opts.NodeID)
-		if err != nil {
-			cleanup()
-			return nil, fmt.Errorf("build publish subject: %w", err)
-		}
-		if err := subscribePlain(publishSubject, handlePublishVolume(provider)); err != nil {
-			cleanup()
-			return nil, err
-		}
-		unpublishSubject, err := ebsprovider.UnpublishSubject(opts.NodeID)
-		if err != nil {
-			cleanup()
-			return nil, fmt.Errorf("build unpublish subject: %w", err)
-		}
-		if err := subscribePlain(unpublishSubject, handleUnpublishVolume(provider)); err != nil {
+		if err := subscribePlain(subject, step.handler); err != nil {
 			cleanup()
 			return nil, err
 		}
@@ -188,7 +184,7 @@ func badRequestError(err error) *ebsprovider.ProviderError {
 }
 
 func versionError(got uint16) *ebsprovider.ProviderError {
-	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: fmt.Sprintf("unsupported schema version %d; supported versions are %d and %d", got, ebsprovider.LegacySchemaVersion, ebsprovider.SchemaVersion)}
+	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: fmt.Sprintf("unsupported schema version %d; %s", got, ebsprovider.SupportedVersionsText())}
 }
 
 func invalidArgumentError(format string, args ...any) *ebsprovider.ProviderError {
@@ -202,6 +198,7 @@ func internalError(format string, args ...any) *ebsprovider.ProviderError {
 // classifyError maps a provider error to the wire ProviderError shape,
 // using only the sentinel errors and ErrorCode values errors.go exports:
 // it has no forward (error -> ProviderError) mapper of its own to call.
+// The provider picks the sentinel for the request's version (HeldElsewhereError).
 func classifyError(err error) *ebsprovider.ProviderError {
 	switch {
 	case errors.Is(err, ebsprovider.ErrAlreadyExists):
@@ -212,6 +209,8 @@ func classifyError(err error) *ebsprovider.ProviderError {
 		return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeNotFound, Message: err.Error()}
 	case errors.Is(err, ebsprovider.ErrUnsupportedVersion):
 		return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: err.Error()}
+	case errors.Is(err, ebsprovider.ErrVolumeHeldElsewhere):
+		return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeHeldElsewhere, Message: err.Error()}
 	case errors.Is(err, ebsprovider.ErrVolumeInUse):
 		return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: err.Error()}
 	case errors.Is(err, ebsprovider.ErrUnsupportedCapability):
@@ -226,178 +225,202 @@ func classifyError(err error) *ebsprovider.ProviderError {
 func handleCapabilities(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.GetCapabilitiesRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		resp, err := provider.GetCapabilities(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, *resp)
+		reply := *resp
+		reply.Versioned = v
+		respond(ctx, msg, reply)
 	}
 }
 
 func handleCreateVolume(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.CreateVolumeRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		vol, err := provider.CreateVolume(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: vol})
+		respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Volume: vol})
 	}
 }
 
 func handleGetVolume(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.GetVolumeRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		vol, err := provider.GetVolume(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: vol})
+		respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Volume: vol})
 	}
 }
 
 func handleListVolumes(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.ListVolumesRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		resp, err := provider.ListVolumes(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, *resp)
+		reply := *resp
+		reply.Versioned = v
+		respond(ctx, msg, reply)
 	}
 }
 
 func handleListSnapshots(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.ListSnapshotsRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		resp, err := provider.ListSnapshots(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, *resp)
+		reply := *resp
+		reply.Versioned = v
+		respond(ctx, msg, reply)
 	}
 }
 
 func handleExpandVolume(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.ExpandVolumeRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		vol, err := provider.ExpandVolume(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: vol})
+		respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Volume: vol})
 	}
 }
 
 func handleDeleteVolume(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.DeleteVolumeRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		if err := provider.DeleteVolume(ctx, req); err != nil {
-			respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned()})
+		respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v})
 	}
 }
 
 func handleDeleteSnapshot(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.DeleteSnapshotRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		if err := provider.DeleteSnapshot(ctx, req); err != nil {
-			respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned()})
+		respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v})
 	}
 }
 
 func handleCopySnapshot(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.CopySnapshotRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		snap, err := provider.CopySnapshot(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Snapshot: snap})
+		respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Snapshot: snap})
 	}
 }
 
@@ -407,12 +430,14 @@ func handleCopySnapshot(provider ebsprovider.EBSProvider) msgHandler {
 func handleCreateSnapshot(nc *nats.Conn, provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.CreateSnapshotRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 
@@ -420,23 +445,23 @@ func handleCreateSnapshot(nc *nats.Conn, provider ebsprovider.EBSProvider) msgHa
 		// token (SnapshotCreateSubjectPrefix + volumeID); a mismatch with
 		// the request body means the two disagree about which volume this create is for.
 		if subjectVolumeID, ok := strings.CutPrefix(msg.Subject, ebsprovider.SnapshotCreateSubjectPrefix); ok && subjectVolumeID != req.VolumeID {
-			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("volume id %q in subject does not match request volume id %q", subjectVolumeID, req.VolumeID)})
+			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: invalidArgumentError("volume id %q in subject does not match request volume id %q", subjectVolumeID, req.VolumeID)})
 			return
 		}
 
 		completionSubject, err := ebsprovider.SnapshotCompletionSubject(req.SnapshotID)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
 		operationID, err := newOperationID()
 		if err != nil {
-			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("generate operation id: %v", err)})
+			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: internalError("generate operation id: %v", err)})
 			return
 		}
 
 		respond(ctx, msg, ebsprovider.CreateSnapshotResponse{
-			Versioned:         ebsprovider.NewVersioned(),
+			Versioned:         v,
 			OperationID:       operationID,
 			CompletionSubject: completionSubject,
 			Snapshot: &ebsprovider.Snapshot{
@@ -452,7 +477,7 @@ func handleCreateSnapshot(nc *nats.Conn, provider ebsprovider.EBSProvider) msgHa
 
 func completeCreateSnapshot(ctx context.Context, nc *nats.Conn, provider ebsprovider.EBSProvider, req ebsprovider.CreateSnapshotRequest, operationID, completionSubject string) {
 	completion := ebsprovider.CreateSnapshotResponse{
-		Versioned:         ebsprovider.NewVersioned(),
+		Versioned:         ebsprovider.ReplyVersioned(req.SchemaVersion),
 		OperationID:       operationID,
 		CompletionSubject: completionSubject,
 	}
@@ -488,27 +513,29 @@ func newOperationID() (string, error) {
 func handlePublishVolume(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.PublishVolumeRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		pub, err := provider.PublishVolume(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
 		if pub == nil {
-			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("provider returned no published volume for %s", req.VolumeID)})
+			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: internalError("provider returned no published volume for %s", req.VolumeID)})
 			return
 		}
 		// An unattachable export is the provider's fault, so it is reported as
 		// internal rather than relayed for the client to reject.
 		if err := pub.Export.Validate(); err != nil {
-			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("provider returned an invalid export for %s: %v", req.VolumeID, err)})
+			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: internalError("provider returned an invalid export for %s: %v", req.VolumeID, err)})
 			return
 		}
 		if req.SchemaVersion == ebsprovider.LegacySchemaVersion {
@@ -517,36 +544,65 @@ func handlePublishVolume(provider ebsprovider.EBSProvider) msgHandler {
 				// A v1 caller cannot safely consume a block export. Do not emit an
 				// empty legacy URI and leave the old client to misattach it.
 				respond(ctx, msg, ebsprovider.PublishVolumeResponseV1{
-					Versioned: ebsprovider.NewVersioned(),
+					Versioned: v,
 					Error:     &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: err.Error()},
 				})
 				return
 			}
 			respond(ctx, msg, ebsprovider.PublishVolumeResponseV1{
-				Versioned: ebsprovider.NewVersioned(),
+				Versioned: v,
 				Published: legacy,
 			})
 			return
 		}
-		respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewLatestVersioned(), Published: pub})
+		respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Published: pub})
 	}
 }
 
 func handleUnpublishVolume(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.UnpublishVolumeRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: badRequestError(err)})
+			return
+		}
+		if err := ebsprovider.CheckUnpublishVersion(req); err != nil {
+			respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: classifyError(err)})
+			return
+		}
+		result, err := provider.UnpublishVolume(ctx, req)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: classifyError(err)})
+			return
+		}
+		response := ebsprovider.UnpublishVolumeResponse{Versioned: v}
+		if result != nil {
+			response.TornDown = result.TornDown
+		}
+		respond(ctx, msg, response)
+	}
+}
+
+func handleHoldsUnsealedWrites(provider ebsprovider.EBSProvider) msgHandler {
+	return func(ctx context.Context, msg *nats.Msg) {
+		var req ebsprovider.HoldsUnsealedWritesRequest
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
-		if err := provider.UnpublishVolume(ctx, req); err != nil {
-			respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+		held, err := provider.HoldsUnsealedWrites(ctx, req.NodeID, req.VolumeIDs)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned()})
+		respond(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Held: held})
 	}
 }

spinifex/providers/ebs/provider.go

@@ -19,10 +19,20 @@ const (
 	// evidence that the peer supports a newer representation.
 	LegacySchemaVersion uint16 = 1
 
+	// DescriptorSchemaVersion is the first version whose PublishVolume reply
+	// carries a VolumeExport descriptor, and so the first that can describe a
+	// block export.
+	DescriptorSchemaVersion uint16 = 2
+
+	// NoSealSchemaVersion is the first version that carries
+	// UnpublishVolumeRequest.NoSeal. A provider that does not serve it refuses
+	// the request as unsupported rather than dropping the flag and sealing.
+	NoSealSchemaVersion uint16 = 3
+
 	// SchemaVersion is the newest wire-contract version understood by this
 	// build. Every request and response carries a version so an unsupported
 	// representation fails at the boundary rather than silently decoding it.
-	SchemaVersion uint16 = 2
+	SchemaVersion uint16 = NoSealSchemaVersion
 )
 
 // Versioned is embedded in every request and response sent over NATS.
@@ -41,14 +51,39 @@ func NewVersioned() Versioned { return Versioned{SchemaVersion: LegacySchemaVers
 func NewLatestVersioned() Versioned { return Versioned{SchemaVersion: SchemaVersion} }
 
 // SupportsSchemaVersion reports whether this build can safely decode version.
+// Every version from LegacySchemaVersion to SchemaVersion is served.
 func SupportsSchemaVersion(version uint16) bool {
-	return version == LegacySchemaVersion || version == SchemaVersion
+	return version >= LegacySchemaVersion && version <= SchemaVersion
+}
+
+// SupportedSchemaVersions lists, in ascending order, every contract version
+// this build serves. GetCapabilitiesResponse.ContractVersions reports it.
+func SupportedSchemaVersions() []uint16 {
+	versions := make([]uint16, 0, SchemaVersion-LegacySchemaVersion+1)
+	for v := LegacySchemaVersion; v <= SchemaVersion; v++ {
+		versions = append(versions, v)
+	}
+	return versions
+}
+
+// ReplyVersioned stamps a reply with its request's version, so a body is never
+// labelled newer than its shape. A version this build cannot serve is answered
+// at LegacySchemaVersion, the one every peer decodes.
+func ReplyVersioned(requested uint16) Versioned {
+	if !SupportsSchemaVersion(requested) {
+		return NewVersioned()
+	}
+	return Versioned{SchemaVersion: requested}
 }
 
 // EBSProvider is the block-storage controller contract consumed by Spinifex.
 // Its operation and idempotency shape follows the CSI controller service, but
 // the transport remains NATS request/reply rather than CSI's gRPC transport.
 type EBSProvider interface {
+	// Required of every provider: a caller counts no answer as held, which
+	// would keep an instance from ever starting on another node.
+	UnsealedWritesChecker
+
 	GetCapabilities(context.Context, GetCapabilitiesRequest) (*GetCapabilitiesResponse, error)
 	CreateVolume(context.Context, CreateVolumeRequest) (*Volume, error)
 	GetVolume(context.Context, GetVolumeRequest) (*Volume, error)
@@ -60,7 +95,7 @@ type EBSProvider interface {
 	CopySnapshot(context.Context, CopySnapshotRequest) (*Snapshot, error)
 	ListSnapshots(context.Context, ListSnapshotsRequest) (*ListSnapshotsResponse, error)
 	PublishVolume(context.Context, PublishVolumeRequest) (*PublishedVolume, error)
-	UnpublishVolume(context.Context, UnpublishVolumeRequest) error
+	UnpublishVolume(context.Context, UnpublishVolumeRequest) (*UnpublishedVolume, error)
 }
 
 // Provider is retained as a concise alias for implementation code. New
@@ -164,8 +199,14 @@ type GetCapabilitiesRequest struct{ Versioned }
 type GetCapabilitiesResponse struct {
 	Versioned
 
-	Capabilities Capabilities   `json:"capabilities"`
-	Error        *ProviderError `json:"error,omitempty"`
+	Capabilities Capabilities `json:"capabilities"`
+
+	// ContractVersions lists every schema version the answering provider
+	// serves, ascending. A provider that predates the field leaves it empty,
+	// which a caller must read as no evidence, not as version 1 only.
+	ContractVersions []uint16 `json:"contract_versions,omitempty"`
+
+	Error *ProviderError `json:"error,omitempty"`
 }
 
 // CapacityRange mirrors CSI's capacity-range semantics. RequiredBytes is the
@@ -563,7 +604,7 @@ func (p *PublishedVolume) AsV1() (*PublishedVolumeV1, error) {
 		return nil, err
 	}
 	if p.Export.Kind != ExportKindNBD {
-		return nil, fmt.Errorf("%s export requires schema version %d", p.Export.Kind, SchemaVersion)
+		return nil, fmt.Errorf("%s export requires schema version %d", p.Export.Kind, DescriptorSchemaVersion)
 	}
 	return &PublishedVolumeV1{
 		VolumeID: p.VolumeID,
@@ -617,6 +658,31 @@ type UnpublishVolumeRequest struct {
 	VolumeID string `json:"volume_id"`
 	Handle   string `json:"handle,omitempty"`
 	NodeID   string `json:"node_id"`
+
+	// NoSeal drops this node's copy unsealed, because a newer owner's supersedes
+	// it; unlike AWS DetachVolume Force, which leaves a copy that must seal.
+	// Carried only at NoSealSchemaVersion or later (CheckUnpublishVersion).
+	NoSeal bool `json:"no_seal,omitempty"`
+}
+
+// CheckUnpublishVersion refuses a request this build cannot serve, and a
+// NoSeal sent below NoSealSchemaVersion: a provider at that version would
+// drop the flag and seal, so no provider may honour it there either.
+func CheckUnpublishVersion(req UnpublishVolumeRequest) error {
+	if err := checkVersion(req.SchemaVersion); err != nil {
+		return err
+	}
+	if req.NoSeal && req.SchemaVersion < NoSealSchemaVersion {
+		return fmt.Errorf("%w: no_seal requires schema version %d, got %d", ErrInvalidArgument, NoSealSchemaVersion, req.SchemaVersion)
+	}
+	return nil
+}
+
+// UnpublishedVolume is the outcome of an UnpublishVolume that succeeded.
+// TornDown reports whether an export existed and was removed; it is reported
+// only at NoSealSchemaVersion or later, and is false at earlier versions.
+type UnpublishedVolume struct {
+	TornDown bool
 }
 
 // UnpublishVolumeResponse is the reply to UnpublishVolumeRequest. A nil Error
@@ -624,13 +690,14 @@ type UnpublishVolumeRequest struct {
 type UnpublishVolumeResponse struct {
 	Versioned
 
-	Error *ProviderError `json:"error,omitempty"`
+	// TornDown mirrors UnpublishedVolume.TornDown.
+	TornDown bool           `json:"torn_down,omitempty"`
+	Error    *ProviderError `json:"error,omitempty"`
 }
 
 // UnsealedWritesChecker answers, for one node, which of a set of volumes it
-// still holds writes for that the backend may not have. It is separate from
-// EBSProvider because only the instance start path asks, and only of the node
-// that last ran the instance.
+// still holds writes for that the backend may not have. A write-through
+// provider answers with none. Consumers that only ask this depend on it alone.
 type UnsealedWritesChecker interface {
 	HoldsUnsealedWrites(ctx context.Context, nodeID string, volumeIDs []string) ([]string, error)
 }
@@ -652,3 +719,25 @@ type HoldsUnsealedWritesResponse struct {
 	Held  []string       `json:"held,omitempty"`
 	Error *ProviderError `json:"error,omitempty"`
 }
+
+// FenceNotice announces that a provider revoked a live export on NodeID
+// because another node now holds the volume, so that node's daemon can stop
+// the guest using it. Sent over FenceNoticeSubject until acknowledged.
+type FenceNotice struct {
+	Versioned
+
+	VolumeID string `json:"volume_id"`
+	NodeID   string `json:"node_id"`
+	Winner   string `json:"winner,omitempty"`
+	Reason   string `json:"reason,omitempty"`
+}
+
+// FenceNoticeAck is the daemon's answer to a FenceNotice. Any answer ends the
+// sender's retries; InstanceID names the guest stopped, if one was, and Error
+// says why the daemon could not act on the notice.
+type FenceNoticeAck struct {
+	Versioned
+
+	InstanceID string         `json:"instance_id,omitempty"`
+	Error      *ProviderError `json:"error,omitempty"`
+}

spinifex/providers/ebs/telemetry.go

@@ -46,6 +46,8 @@ const (
 	verbVolumeUnpublish = "volume.unpublish"
 	verbSnapshotDelete  = "snapshot.delete"
 	verbSnapshotList    = "snapshot.list"
+	verbUnsealedWrites  = "volume.unsealed"
+	verbFenceNotice     = "volume.fenced"
 )
 
 // subjectVerbs maps every fixed subject to its verb. The dynamic subject
@@ -107,10 +109,16 @@ func parseSubject(subject string) subjectTarget {
 		return subjectTarget{verb: VerbUnknown}
 	}
 	switch suffix {
-	case "mount":
+	case publishVerb:
 		return subjectTarget{verb: verbVolumePublish, nodeID: nodeID}
-	case "unmount":
+	case unpublishVerb:
 		return subjectTarget{verb: verbVolumeUnpublish, nodeID: nodeID}
+	case unsealedVerb:
+		return subjectTarget{verb: verbUnsealedWrites, nodeID: nodeID}
+	case nodeCapabilitiesVerb:
+		return subjectTarget{verb: verbCapabilities, nodeID: nodeID}
+	case fenceNoticeVerb:
+		return subjectTarget{verb: verbFenceNotice, nodeID: nodeID}
 	default:
 		return subjectTarget{verb: VerbUnknown}
 	}

spinifex/providers/ebs/telemetry_test.go

@@ -30,6 +30,12 @@ func TestSubjectVerb_StripsIDsFromEveryDynamicSubject(t *testing.T) {
 	require.NoError(t, err)
 	unpublish, err := UnpublishSubject(nodeID)
 	require.NoError(t, err)
+	unsealed, err := UnsealedWritesSubject(nodeID)
+	require.NoError(t, err)
+	nodeCapabilities, err := NodeCapabilitiesSubject(nodeID)
+	require.NoError(t, err)
+	fenced, err := FenceNoticeSubject(nodeID)
+	require.NoError(t, err)
 
 	cases := map[string]string{
 		CapabilitiesSubject:   verbCapabilities,
@@ -47,6 +53,9 @@ func TestSubjectVerb_StripsIDsFromEveryDynamicSubject(t *testing.T) {
 		snapshotCreate:        verbSnapshotCreate,
 		publish:               verbVolumePublish,
 		unpublish:             verbVolumeUnpublish,
+		unsealed:              verbUnsealedWrites,
+		nodeCapabilities:      verbCapabilities,
+		fenced:                verbFenceNotice,
 	}
 
 	for subject, want := range cases {

spinifex/providers/ebs/viperblock/conformance_test.go

@@ -15,6 +15,7 @@ import (
 	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs/conformance"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	testpredastore "github.com/mulgadc/spinifex/tests/fixtures/predastore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/require"
@@ -61,7 +62,8 @@ func requireExportTools(t *testing.T) string {
 // would go to a subject nobody serves and time out instead of being refused.
 // Both share the NATS the volume leases live in and the bucket the volumes
 // live in, so a second opener is refused by the lease, not by the harness.
-func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider, string, string) {
+// The last return kills a published export's nbdkit behind the handlers' back.
+func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider, string, string, func(*testing.T, *ebsprovider.PublishedVolume)) {
 	t.Helper()
 	pluginPath := requireExportTools(t)
 
@@ -98,10 +100,21 @@ func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider
 	_, err = js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
 	require.NoError(t, err)
 
+	killExport := func(t *testing.T, published *ebsprovider.PublishedVolume) {
+		t.Helper()
+		for _, node := range []*Config{first, second} {
+			if mv, ok := findMountedVolume(node, published.VolumeID); ok && node.NodeName == published.NodeID {
+				require.NoError(t, hostprocess.ForceKillProcess(mv.PID, 5*time.Second))
+				return
+			}
+		}
+		t.Fatalf("no mounted entry for %s on %s", published.VolumeID, published.NodeID)
+	}
+
 	return func(t *testing.T) ebsprovider.EBSProvider {
 		t.Helper()
 		return ebsprovider.NewNATSProvider(nc, 60*time.Second)
-	}, first.NodeName, second.NodeName
+	}, first.NodeName, second.NodeName, killExport
 }
 
 // runPrefix names one run's volumes so it cannot meet an earlier run's
@@ -113,11 +126,12 @@ func runPrefix() string {
 // TestViperblockdConformance judges the provider handlers by the same suite
 // MemoryProvider and qemunbdd answer to.
 func TestViperblockdConformance(t *testing.T) {
-	newProvider, nodeID, otherNodeID := inProcessProvider(t)
+	newProvider, nodeID, otherNodeID, killExport := inProcessProvider(t)
 	conformance.RunSuiteWithConfig(t, newProvider, conformance.SuiteConfig{
 		NamePrefix:  runPrefix(),
 		NodeID:      nodeID,
 		OtherNodeID: otherNodeID,
+		KillExport:  killExport,
 	})
 }
 
@@ -125,7 +139,7 @@ func TestViperblockdConformance(t *testing.T) {
 // cannot: whether the nbdkit export viperblockd publishes is usable by libnbd,
 // which knows nothing about this codebase.
 func TestViperblockdNBDClient(t *testing.T) {
-	newProvider, nodeID, _ := inProcessProvider(t)
+	newProvider, nodeID, _, _ := inProcessProvider(t)
 	conformance.RunNBDClientSuiteWithConfig(t, newProvider, conformance.NBDClientConfig{
 		NodeID:       nodeID,
 		VolumePrefix: "vol-" + runPrefix() + "nbd",

spinifex/providers/ebs/viperblock/provider_contract_test.go

@@ -0,0 +1,339 @@
+package viperblock
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"maps"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/testkit/deadpid"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
+	"github.com/mulgadc/viperblock/viperblock"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestProviderHandlers_RepliesInRequestVersion holds every subject to the
+// rule a mixed-version rollout depends on: a caller at version N must get a
+// reply at N, success or error, or it may not be able to parse it.
+func TestProviderHandlers_RepliesInRequestVersion(t *testing.T) {
+	prevProbe := ownerProbeTimeout
+	ownerProbeTimeout = 100 * time.Millisecond
+	t.Cleanup(func() { ownerProbeTimeout = prevProbe })
+
+	_, natsURL := setupEmbeddedNATS(t)
+	cfg := setupTestConfig(t, natsURL)
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	for _, version := range ebsprovider.SupportedSchemaVersions() {
+		for _, tc := range providerSubjectCases() {
+			t.Run(fmt.Sprintf("v%d/%s", version, tc.name), func(t *testing.T) {
+				body := map[string]any{"schema_version": version}
+				maps.Copy(body, tc.body)
+				msg := requestProvider(t, nc, tc.subject, marshalRequest(t, body))
+
+				var resp versionedErrorResponse
+				require.NoError(t, json.Unmarshal(msg.Data, &resp))
+				assert.Equal(t, version, resp.SchemaVersion)
+			})
+		}
+	}
+}
+
+// TestProviderHandlers_UnpublishNoSealNeverSeals is the reason no_seal exists:
+// a node whose copy is behind a newer owner's must drop it, and a seal would
+// write that stale copy over the owner's block map.
+func TestProviderHandlers_UnpublishNoSealNeverSeals(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-unpubnoseal01"
+	sealed := false
+	cfg := fencedConfig(t, natsURL, "test-node", volumeName)
+	withLocalState(t, cfg, volumeName)
+	cfg.sealVolume = func(context.Context, string) error { sealed = true; return nil }
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	subject, err := ebsprovider.UnpublishSubject(cfg.NodeName)
+	require.NoError(t, err)
+	body := marshalRequest(t, map[string]any{
+		"schema_version": ebsprovider.NoSealSchemaVersion,
+		"volume_id":      volumeName,
+		"node_id":        cfg.NodeName,
+		"no_seal":        true,
+	})
+	msg := requestProvider(t, nc, subject, body)
+
+	var resp ebsprovider.UnpublishVolumeResponse
+	require.NoError(t, json.Unmarshal(msg.Data, &resp))
+	require.Nil(t, resp.Error)
+	assert.True(t, resp.TornDown, "the volume was mounted, so the export was torn down")
+	assert.False(t, sealed, "a no_seal unpublish must never seal")
+	_, mounted := findMountedVolume(cfg, volumeName)
+	assert.False(t, mounted, "the entry must go with the export")
+}
+
+// TestProviderHandlers_UnpublishNoSealBelowV3Refused covers the other half: a
+// caller below version 3 cannot ask for no_seal, so the flag is refused rather
+// than ignored, and the export is left alone.
+func TestProviderHandlers_UnpublishNoSealBelowV3Refused(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-unpubnosealv2"
+	sealed := false
+	cfg := fencedConfig(t, natsURL, "test-node", volumeName)
+	cfg.sealVolume = func(context.Context, string) error { sealed = true; return nil }
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	subject, err := ebsprovider.UnpublishSubject(cfg.NodeName)
+	require.NoError(t, err)
+	body := marshalRequest(t, map[string]any{
+		"schema_version": ebsprovider.DescriptorSchemaVersion,
+		"volume_id":      volumeName,
+		"node_id":        cfg.NodeName,
+		"no_seal":        true,
+	})
+	msg := requestProvider(t, nc, subject, body)
+
+	var resp ebsprovider.UnpublishVolumeResponse
+	require.NoError(t, json.Unmarshal(msg.Data, &resp))
+	require.NotNil(t, resp.Error)
+	assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, resp.Error.Code)
+	assert.False(t, sealed)
+	_, mounted := findMountedVolume(cfg, volumeName)
+	assert.True(t, mounted, "a refused unpublish must leave the export running")
+}
+
+// TestProviderHandlers_PublishVolume_DeadEntryIsNotHandedBack pins the removed
+// early return: an entry whose nbdkit has died must be dropped and remounted,
+// never answered with a URI nothing serves.
+func TestProviderHandlers_PublishVolume_DeadEntryIsNotHandedBack(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+	cfg := setupTestConfig(t, natsURL)
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	const volumeName = "vol-pubdeadentry1"
+	const deadURI = "nbd://127.0.0.1:1"
+	cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{
+		Name: volumeName, VB: createTestVBWithState(t, volumeName), PID: deadpid.PID, NBDURI: deadURI,
+	})
+
+	subject, err := ebsprovider.PublishSubject(cfg.NodeName)
+	require.NoError(t, err)
+	body := marshalRequest(t, map[string]any{
+		"schema_version": ebsprovider.SchemaVersion,
+		"volume_id":      volumeName,
+		"node_id":        cfg.NodeName,
+	})
+	msg := requestProvider(t, nc, subject, body)
+
+	var resp ebsprovider.PublishVolumeResponse
+	require.NoError(t, json.Unmarshal(msg.Data, &resp))
+	assert.True(t, resp.Error != nil || resp.Published != nil, "the reply must be a refusal or an export")
+	assert.NotContains(t, string(msg.Data), deadURI, "the dead entry's URI was handed back")
+	mv, _ := findMountedVolume(cfg, volumeName)
+	assert.NotEqual(t, deadURI, mv.NBDURI, "the dead entry must not survive the republish")
+}
+
+// TestProviderHandlers_PublishVolume_RepublishRepliesInRequestVersion checks a
+// successful publish reply, not only refusals, at every served version: a
+// republish of a live export is the success path that needs no nbdkit.
+func TestProviderHandlers_PublishVolume_RepublishRepliesInRequestVersion(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+	cfg := setupTestConfig(t, natsURL)
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	const volumeName = "vol-republishver1"
+	socket := filepath.Join(t.TempDir(), "nbd.sock")
+	require.NoError(t, os.WriteFile(socket, nil, 0o600))
+	liveURI := nbd.FormatNBDSocketURI(socket)
+	cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{
+		Name: volumeName, VB: createTestVBWithState(t, volumeName), PID: os.Getpid(), Socket: socket, NBDURI: liveURI,
+	})
+
+	subject, err := ebsprovider.PublishSubject(cfg.NodeName)
+	require.NoError(t, err)
+	for _, version := range ebsprovider.SupportedSchemaVersions() {
+		t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) {
+			msg := requestProvider(t, nc, subject, marshalRequest(t, map[string]any{
+				"schema_version": version,
+				"volume_id":      volumeName,
+				"node_id":        cfg.NodeName,
+			}))
+			var resp versionedErrorResponse
+			require.NoError(t, json.Unmarshal(msg.Data, &resp))
+			require.Nil(t, resp.Error, "the republish must succeed for this to check a success reply")
+			assert.Equal(t, version, resp.SchemaVersion)
+			assert.Contains(t, string(msg.Data), socket, "the live export is handed back")
+		})
+	}
+}
+
+// TestProviderHandlers_NodeCapabilitiesListsServedVersions pins the per-node
+// subject to the same contract versions this build serves.
+func TestProviderHandlers_NodeCapabilitiesListsServedVersions(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+	cfg := setupTestConfig(t, natsURL)
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	subject, err := ebsprovider.NodeCapabilitiesSubject(cfg.NodeName)
+	require.NoError(t, err)
+	msg := requestProvider(t, nc, subject, marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion}))
+
+	var resp ebsprovider.GetCapabilitiesResponse
+	require.NoError(t, json.Unmarshal(msg.Data, &resp))
+	require.Nil(t, resp.Error)
+	assert.Equal(t, ebsprovider.SupportedSchemaVersions(), resp.ContractVersions)
+}
+
+// TestProviderHandlers_UnpublishNoSealAboveServedVersionRefused covers a caller
+// newer than this build: a no_seal it cannot vouch for is refused, and the
+// export is neither torn down nor sealed.
+func TestProviderHandlers_UnpublishNoSealAboveServedVersionRefused(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-unpubnosealv4"
+	sealed := false
+	cfg := fencedConfig(t, natsURL, "test-node", volumeName)
+	withLocalState(t, cfg, volumeName)
+	cfg.sealVolume = func(context.Context, string) error { sealed = true; return nil }
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	subject, err := ebsprovider.UnpublishSubject(cfg.NodeName)
+	require.NoError(t, err)
+	msg := requestProvider(t, nc, subject, marshalRequest(t, map[string]any{
+		"schema_version": ebsprovider.SchemaVersion + 1,
+		"volume_id":      volumeName,
+		"node_id":        cfg.NodeName,
+		"no_seal":        true,
+	}))
+
+	var resp ebsprovider.UnpublishVolumeResponse
+	require.NoError(t, json.Unmarshal(msg.Data, &resp))
+	require.NotNil(t, resp.Error)
+	assert.Equal(t, ebsprovider.ErrorCodeUnsupportedVersion, resp.Error.Code)
+	assert.False(t, resp.TornDown)
+	assert.False(t, sealed, "a refused unpublish must not seal")
+	_, mounted := findMountedVolume(cfg, volumeName)
+	assert.True(t, mounted, "a refused unpublish must leave the export running")
+}
+
+// TestPublishError_Codes pins the code each mount failure reaches a caller as.
+// held_elsewhere is gated on the request version because v1.23.0 callers only
+// know volume_in_use, and a mode mismatch is never held_elsewhere.
+func TestPublishError_Codes(t *testing.T) {
+	leaseHeld := fmt.Errorf("acquire: %w", errVolumeLeaseHeld)
+	cases := []struct {
+		name    string
+		version uint16
+		err     error
+		want    ebsprovider.ErrorCode
+	}{
+		{"absent volume", ebsprovider.SchemaVersion, fmt.Errorf("load: %w", viperblock.ErrStateNotFound), ebsprovider.ErrorCodeNotFound},
+		{"lease held at v1", ebsprovider.LegacySchemaVersion, leaseHeld, ebsprovider.ErrorCodeVolumeInUse},
+		{"lease held at v2", ebsprovider.DescriptorSchemaVersion, leaseHeld, ebsprovider.ErrorCodeVolumeInUse},
+		{"lease held at v3", ebsprovider.NoSealSchemaVersion, leaseHeld, ebsprovider.ErrorCodeVolumeHeldElsewhere},
+		{"mode mismatch at v3", ebsprovider.NoSealSchemaVersion, fmt.Errorf("mounted: %w", errReadOnlyMismatch), ebsprovider.ErrorCodeVolumeInUse},
+		{"backing store not ready", ebsprovider.SchemaVersion, fmt.Errorf("load: %w", viperblock.ErrStateBackendUnavailable), ebsprovider.ErrorCodeUnavailable},
+		{"lease store not ready", ebsprovider.SchemaVersion, fmt.Errorf("open: %w", ErrLeaseStoreUnavailable), ebsprovider.ErrorCodeUnavailable},
+		{"anything else", ebsprovider.SchemaVersion, errors.New("nbdkit exited"), ebsprovider.ErrorCodeInternal},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := publishError(tc.version, "vol-publisherror01", tc.err)
+			require.NotNil(t, got)
+			assert.Equal(t, tc.want, got.Code)
+		})
+	}
+	assert.True(t, ebsprovider.ErrorCodeUnavailable.Retryable(), "a backing store that is not ready must read as retry-later")
+}
+
+// TestProviderHandlers_PublishVolume_StoreNotReadyIsUnavailable carries a mount
+// refused because a store is not ready yet to the wire as retry-later, so a
+// caller waits for the store instead of giving up on the volume.
+func TestProviderHandlers_PublishVolume_StoreNotReadyIsUnavailable(t *testing.T) {
+	for _, cause := range []error{ErrLeaseStoreUnavailable, viperblock.ErrStateBackendUnavailable} {
+		t.Run(cause.Error(), func(t *testing.T) {
+			_, natsURL := setupEmbeddedNATS(t)
+			cfg := setupTestConfig(t, natsURL)
+			cfg.constructVB = func(context.Context, string) (*viperblock.VB, int, error) {
+				return nil, 0, fmt.Errorf("open: %w", cause)
+			}
+			nc := startProviderSubjects(t, cfg, natsURL)
+
+			subject, err := ebsprovider.PublishSubject(cfg.NodeName)
+			require.NoError(t, err)
+			msg := requestProvider(t, nc, subject, marshalRequest(t, map[string]any{
+				"schema_version": ebsprovider.SchemaVersion,
+				"volume_id":      "vol-storenotready",
+				"node_id":        cfg.NodeName,
+			}))
+
+			var resp ebsprovider.PublishVolumeResponse
+			require.NoError(t, json.Unmarshal(msg.Data, &resp))
+			require.NotNil(t, resp.Error)
+			assert.Equal(t, ebsprovider.ErrorCodeUnavailable, resp.Error.Code)
+			assert.Nil(t, resp.Published)
+		})
+	}
+}
+
+// TestProviderHandlers_CreateSnapshot_QueueRefusesWhileLeaseHeldElsewhere is
+// what makes a snapshot from a non-owner safe: whichever node answers the queue
+// subject must not snapshot a checkpoint the holder may still be writing.
+func TestProviderHandlers_CreateSnapshot_QueueRefusesWhileLeaseHeldElsewhere(t *testing.T) {
+	const volumeName = "vol-snapqueueheld1"
+	const snapshotID = "snap-queueheld00001"
+
+	// The volume really exists in the backend, so only the lease stands
+	// between this request and a snapshot written beside the holder's writes.
+	key := testMasterKey(t)
+	backend := newRecordingS3(t, "/"+volumeName+"/config.json", seedVolumeConfig(t, volumeName, key))
+	_, natsURL := setupEmbeddedNATS(t)
+	cfg := setupTestConfig(t, natsURL)
+	cfg.S3Host = backend.URL
+	cfg.masterKey = key
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	holder := newTestLeases(t, natsURL, "node-other")
+	_, err := holder.acquire(t.Context(), volumeName)
+	require.NoError(t, err)
+
+	completionSubject, err := ebsprovider.SnapshotCompletionSubject(snapshotID)
+	require.NoError(t, err)
+	completions := make(chan *nats.Msg, 1)
+	sub, err := nc.ChanSubscribe(completionSubject, completions)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, nc.Flush())
+
+	body := marshalRequest(t, map[string]any{
+		"schema_version": ebsprovider.SchemaVersion,
+		"volume_id":      volumeName,
+		"snapshot_id":    snapshotID,
+	})
+	accepted := requestProvider(t, nc, ebsprovider.SnapshotCreateSubjectPrefix+volumeName, body)
+	var acceptance ebsprovider.CreateSnapshotResponse
+	require.NoError(t, json.Unmarshal(accepted.Data, &acceptance))
+	require.Nil(t, acceptance.Error)
+
+	select {
+	case msg := <-completions:
+		var completion ebsprovider.CreateSnapshotResponse
+		require.NoError(t, json.Unmarshal(msg.Data, &completion))
+		require.NotNil(t, completion.Error, "a snapshot under another node's lease must be refused")
+		assert.Equal(t, ebsprovider.ErrorCodeUnavailable, completion.Error.Code)
+		assert.Nil(t, completion.Snapshot)
+	case <-time.After(10 * time.Second):
+		t.Fatal("no completion published")
+	}
+	assert.Empty(t, backend.matching(func(r string) bool { return strings.HasPrefix(r, "PUT ") && strings.Contains(r, snapshotID) }),
+		"no snapshot may be written while another node holds the volume")
+}

spinifex/providers/ebs/viperblock/provider_handlers.go

@@ -149,6 +149,16 @@ func RegisterProviderSubjects(cfg *Config, nc *nats.Conn) error {
 		return fmt.Errorf("subscribe to %s: %w", unsealedSubject, err)
 	}
 
+	// The queue subject is answered by any node's provider; this one only by
+	// this node's, so a daemon can learn what its own provider serves.
+	nodeCapabilitiesSubject, err := ebsprovider.NodeCapabilitiesSubject(cfg.NodeName)
+	if err != nil {
+		return fmt.Errorf("build node capabilities subject: %w", err)
+	}
+	if _, err := nc.Subscribe(nodeCapabilitiesSubject, tracedProviderHandler(handleProviderCapabilities)); err != nil {
+		return fmt.Errorf("subscribe to %s: %w", nodeCapabilitiesSubject, err)
+	}
+
 	return nil
 }
 
@@ -188,7 +198,7 @@ func badRequestError(err error) *ebsprovider.ProviderError {
 }
 
 func unsupportedVersionError(got uint16) *ebsprovider.ProviderError {
-	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: fmt.Sprintf("unsupported schema version %d; supported versions are %d and %d", got, ebsprovider.LegacySchemaVersion, ebsprovider.SchemaVersion)}
+	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: fmt.Sprintf("unsupported schema version %d; %s", got, ebsprovider.SupportedVersionsText())}
 }
 
 // respondPublishedVolume selects the exact publish response a caller asked
@@ -199,19 +209,19 @@ func respondPublishedVolume(ctx context.Context, msg *nats.Msg, schemaVersion ui
 		legacy, err := published.AsV1()
 		if err != nil {
 			respondProvider(ctx, msg, ebsprovider.PublishVolumeResponseV1{
-				Versioned: ebsprovider.NewVersioned(),
+				Versioned: ebsprovider.ReplyVersioned(schemaVersion),
 				Error:     &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: err.Error()},
 			})
 			return
 		}
 		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponseV1{
-			Versioned: ebsprovider.NewVersioned(),
+			Versioned: ebsprovider.ReplyVersioned(schemaVersion),
 			Published: legacy,
 		})
 		return
 	}
 	respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{
-		Versioned: ebsprovider.NewLatestVersioned(),
+		Versioned: ebsprovider.ReplyVersioned(schemaVersion),
 		Published: published,
 	})
 }
@@ -232,6 +242,14 @@ func volumeInUseError(format string, args ...any) *ebsprovider.ProviderError {
 	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: fmt.Sprintf(format, args...)}
 }
 
+func heldElsewhereError(format string, args ...any) *ebsprovider.ProviderError {
+	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeHeldElsewhere, Message: fmt.Sprintf(format, args...)}
+}
+
+// errReadOnlyMismatch is a remount asking for the other access mode than the
+// running export's, which is fixed when nbdkit starts.
+var errReadOnlyMismatch = errors.New("already mounted with the other access mode")
+
 func internalError(format string, args ...any) *ebsprovider.ProviderError {
 	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: fmt.Sprintf(format, args...)}
 }
@@ -276,18 +294,21 @@ func findMountedVolume(cfg *Config, volumeID string) (MountedVolume, bool) {
 
 func handleProviderCapabilities(ctx context.Context, msg *nats.Msg) {
 	var req ebsprovider.GetCapabilitiesRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.capabilities: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
 		slog.Error("ebs.provider.capabilities: unsupported schema version", "version", req.SchemaVersion)
-		respondProvider(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	respondProvider(ctx, msg, ebsprovider.GetCapabilitiesResponse{
-		Versioned: ebsprovider.NewVersioned(),
+		Versioned:        v,
+		ContractVersions: ebsprovider.SupportedSchemaVersions(),
 		Capabilities: ebsprovider.Capabilities{
 			// Every snapshot freezes the live checkpoint rather than
 			// depending on a guest-coordinated quiesce.
@@ -387,24 +408,26 @@ func listTopLevelPrefixes(ctx context.Context, store objectstore.ObjectStore, bu
 
 func handleListVolumes(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.ListVolumesRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.list: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 
 	ids, err := listVolumePrefixes(ctx, providerObjectStoreFactory(cfg), cfg.Bucket)
 	if err != nil {
 		slog.Error("ebs.provider.volume.list: failed to enumerate volumes", "err", err)
-		respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("enumerate volumes: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: v, Error: internalError("enumerate volumes: %v", err)})
 		return
 	}
 
-	response := ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned()}
+	response := ebsprovider.ListVolumesResponse{Versioned: v}
 	response.Volumes, response.NextToken = ebsprovider.Page(ids, req.StartingToken, int(req.PageSize()),
 		func(id string) ebsprovider.VolumeRef {
 			return ebsprovider.VolumeRef{ID: id, Handle: volumeHandle(id)}
@@ -417,24 +440,26 @@ func handleListVolumes(ctx context.Context, cfg *Config, msg *nats.Msg) {
 // encrypted state, and reading it would mean a fetch per snapshot listed.
 func handleListSnapshots(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.ListSnapshotsRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.snapshot.list: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 
 	ids, err := listSnapshotPrefixes(ctx, providerObjectStoreFactory(cfg), cfg.Bucket)
 	if err != nil {
 		slog.Error("ebs.provider.snapshot.list: failed to enumerate snapshots", "err", err)
-		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("enumerate snapshots: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: internalError("enumerate snapshots: %v", err)})
 		return
 	}
 
-	response := ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned()}
+	response := ebsprovider.ListSnapshotsResponse{Versioned: v}
 	response.Snapshots, response.NextToken = ebsprovider.Page(ids, req.StartingToken, int(req.PageSize()),
 		func(id string) ebsprovider.SnapshotRef {
 			return ebsprovider.SnapshotRef{ID: id, Handle: snapshotHandle(id)}
@@ -474,40 +499,42 @@ func buildProviderVBConfig(cfg *Config, volumeID string, volumeSizeBytes uint64,
 
 func handleCreateVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.CreateVolumeRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.create: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
 		slog.Error("ebs.provider.volume.create: unsupported schema version", "version", req.SchemaVersion)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.VolumeID) {
 		slog.Error("ebs.provider.volume.create: invalid volume id", "volume", req.VolumeID)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
 		return
 	}
 	if req.CapacityRange.RequiredBytes <= 0 || (req.CapacityRange.LimitBytes > 0 && req.CapacityRange.RequiredBytes > req.CapacityRange.LimitBytes) {
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid capacity range")})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid capacity range")})
 		return
 	}
 	if req.SourceSnapshotID != "" && req.SourceSnapshotVolumeID == "" {
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("source_snapshot_volume_id is required with source_snapshot_id")})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: invalidArgumentError("source_snapshot_volume_id is required with source_snapshot_id")})
 		return
 	}
 	if err := ebsprovider.ValidateSeedData(req.SeedData); err != nil {
 		slog.Error("ebs.provider.volume.create: seed data rejected", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("%v", err)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: invalidArgumentError("%v", err)})
 		return
 	}
 	if int64(len(req.SeedData)) > req.CapacityRange.RequiredBytes {
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("seed data is larger than the requested capacity")})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: invalidArgumentError("seed data is larger than the requested capacity")})
 		return
 	}
 	if len(req.SeedData) > 0 && req.SourceSnapshotID != "" {
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("seed data cannot be combined with a source snapshot")})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: invalidArgumentError("seed data cannot be combined with a source snapshot")})
 		return
 	}
 
@@ -516,16 +543,16 @@ func handleCreateVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	case err == nil:
 		if existing.CapacityBytes != req.CapacityRange.RequiredBytes {
 			slog.Error("ebs.provider.volume.create: volume exists with different capacity", "volume", req.VolumeID)
-			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: alreadyExistsError("volume %s already exists with a different capacity", req.VolumeID)})
+			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: alreadyExistsError("volume %s already exists with a different capacity", req.VolumeID)})
 			return
 		}
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: existing})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Volume: existing})
 		return
 	case errors.Is(err, viperblock.ErrStateNotFound):
 		// Volume does not exist yet: fall through and create it.
 	default:
 		slog.Error("ebs.provider.volume.create: existence check failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("check existing volume: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: internalError("check existing volume: %v", err)})
 		return
 	}
 
@@ -534,7 +561,7 @@ func handleCreateVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	vb, err := viperblock.New(vbconfig, "s3", s3cfg)
 	if err != nil {
 		slog.Error("ebs.provider.volume.create: new viperblock failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("new viperblock: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: internalError("new viperblock: %v", err)})
 		return
 	}
 	defer vb.Detach()
@@ -542,26 +569,26 @@ func handleCreateVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 
 	if err := vb.Backend.InitCtx(ctx); err != nil {
 		slog.Error("ebs.provider.volume.create: backend init failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("backend init: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: internalError("backend init: %v", err)})
 		return
 	}
 	if req.SourceSnapshotID != "" {
 		if perr := verifySourceSnapshot(vb, req.SourceSnapshotID, req.SourceSnapshotVolumeID); perr != nil {
 			slog.Error("ebs.provider.volume.create: source snapshot rejected", "volume", req.VolumeID, "snapshot", req.SourceSnapshotID, "err", perr.Message)
-			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: perr})
+			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: perr})
 			return
 		}
 	}
 	if len(req.SeedData) > 0 {
 		if err := seedVolume(ctx, vb, req.SeedData); err != nil {
 			slog.Error("ebs.provider.volume.create: seed failed", "volume", req.VolumeID, "bytes", len(req.SeedData), "err", err)
-			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("seed volume: %v", err)})
+			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: internalError("seed volume: %v", err)})
 			return
 		}
 	} else {
 		if err := vb.SaveStateCtx(ctx); err != nil {
 			slog.Error("ebs.provider.volume.create: save state failed", "volume", req.VolumeID, "err", err)
-			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("save state: %v", err)})
+			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: internalError("save state: %v", err)})
 			return
 		}
 		// A state read prefers a local copy over the backend, so state left on
@@ -574,7 +601,7 @@ func handleCreateVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 
 	slog.Info("ebs.provider.volume.create: created", "volume", req.VolumeID, "capacityBytes", req.CapacityRange.RequiredBytes, "seedBytes", len(req.SeedData))
 	respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{
-		Versioned: ebsprovider.NewVersioned(),
+		Versioned: v,
 		Volume: &ebsprovider.Volume{
 			ID:            req.VolumeID,
 			CapacityBytes: req.CapacityRange.RequiredBytes,
@@ -667,72 +694,76 @@ func describeVolumeEngine(ctx context.Context, cfg *Config, volumeID string) (*e
 
 func handleGetVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.GetVolumeRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.describe: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.VolumeID) {
-		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
 		return
 	}
 
 	volume, err := describeVolumeEngine(ctx, cfg, req.VolumeID)
 	if err != nil {
 		if errors.Is(err, viperblock.ErrStateNotFound) {
-			respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: notFoundError("volume %s not found", req.VolumeID)})
+			respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: notFoundError("volume %s not found", req.VolumeID)})
 			return
 		}
 		slog.Error("ebs.provider.volume.describe: failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("describe volume: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: internalError("describe volume: %v", err)})
 		return
 	}
-	respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: volume})
+	respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Volume: volume})
 }
 
 func handleExpandVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.ExpandVolumeRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.expand: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.VolumeID) {
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
 		return
 	}
 	if req.CapacityRange.RequiredBytes <= 0 {
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid capacity range")})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid capacity range")})
 		return
 	}
 
 	if mv, ok := findMountedVolume(cfg, req.VolumeID); ok && mv.VB != nil {
 		slog.Error("ebs.provider.volume.expand: volume is mounted", "volume", req.VolumeID)
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is mounted; detach before expanding", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: volumeInUseError("volume %s is mounted; detach before expanding", req.VolumeID)})
 		return
 	}
 
 	vb, lease, err := openVolumeVB(ctx, cfg, req.VolumeID)
 	if err != nil {
 		if errors.Is(err, viperblock.ErrStateNotFound) {
-			respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: notFoundError("volume %s not found", req.VolumeID)})
+			respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: notFoundError("volume %s not found", req.VolumeID)})
 			return
 		}
 		if errors.Is(err, errVolumeLeaseHeld) {
 			slog.Error("ebs.provider.volume.expand: volume is leased elsewhere", "volume", req.VolumeID, "err", err)
-			respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is open elsewhere: %v", req.VolumeID, err)})
+			respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: volumeInUseError("volume %s is open elsewhere: %v", req.VolumeID, err)})
 			return
 		}
 		slog.Error("ebs.provider.volume.expand: open volume failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("open volume: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: internalError("open volume: %v", err)})
 		return
 	}
 	defer func() {
@@ -742,7 +773,7 @@ func handleExpandVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 
 	currentBytes := safecast.Uint64ToInt64(vb.GetVolumeSize())
 	if req.CapacityRange.RequiredBytes < currentBytes {
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("volume expansion is grow-only")})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: invalidArgumentError("volume expansion is grow-only")})
 		return
 	}
 
@@ -751,18 +782,18 @@ func handleExpandVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	rawConfig, err := json.Marshal(vc)
 	if err != nil {
 		slog.Error("ebs.provider.volume.expand: marshal volume config failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("marshal volume config: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: internalError("marshal volume config: %v", err)})
 		return
 	}
 	if err := applyConfigUpdate(ctx, vb, rawConfig); err != nil {
 		slog.Error("ebs.provider.volume.expand: apply config update failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("apply config update: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: internalError("apply config update: %v", err)})
 		return
 	}
 
 	slog.Info("ebs.provider.volume.expand: expanded", "volume", req.VolumeID, "capacityBytes", req.CapacityRange.RequiredBytes)
 	respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{
-		Versioned: ebsprovider.NewVersioned(),
+		Versioned: v,
 		Volume: &ebsprovider.Volume{
 			ID:            req.VolumeID,
 			CapacityBytes: safecast.Uint64ToInt64(vb.GetVolumeSize()),
@@ -856,17 +887,19 @@ func deleteObjectPrefix(ctx context.Context, store objectstore.ObjectStore, buck
 
 func handleDeleteVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *nats.Msg) {
 	var req ebsprovider.DeleteVolumeRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.delete: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.VolumeID) {
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
 		return
 	}
 
@@ -880,7 +913,7 @@ func handleDeleteVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *na
 	}
 	if mountedHere || ownership == ownershipMounted {
 		slog.Error("ebs.provider.volume.delete: volume is published", "volume", req.VolumeID)
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is published; unpublish it before deleting", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: volumeInUseError("volume %s is published; unpublish it before deleting", req.VolumeID)})
 		return
 	}
 	// Ownership could not be established. Deleting is irreversible, so an
@@ -888,7 +921,7 @@ func handleDeleteVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *na
 	// retry once the owner is reachable.
 	if ownership == ownershipUnknown {
 		slog.Error("ebs.provider.volume.delete: could not establish whether the volume is published", "volume", req.VolumeID)
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unavailableError("volume %s: could not establish whether it is published; retry when its owner is reachable", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: unavailableError("volume %s: could not establish whether it is published; retry when its owner is reachable", req.VolumeID)})
 		return
 	}
 
@@ -908,44 +941,46 @@ func handleDeleteVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *na
 	store := providerObjectStoreFactory(cfg)
 	if err := deleteObjectPrefix(ctx, store, cfg.Bucket, req.VolumeID+"-efi/"); err != nil {
 		slog.Error("ebs.provider.volume.delete: failed to delete aux objects", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("delete aux objects: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: internalError("delete aux objects: %v", err)})
 		return
 	}
 	if err := deleteObjectPrefix(ctx, store, cfg.Bucket, req.VolumeID+"/"); err != nil {
 		slog.Error("ebs.provider.volume.delete: failed to delete volume objects", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("delete volume objects: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: internalError("delete volume objects: %v", err)})
 		return
 	}
 
 	slog.Info("ebs.provider.volume.delete: deleted", "volume", req.VolumeID)
-	respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned()})
+	respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v})
 }
 
 func handleDeleteSnapshot(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.DeleteSnapshotRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.snapshot.delete: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.SnapshotID) {
-		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid snapshot id %q", req.SnapshotID)})
+		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: invalidArgumentError("invalid snapshot id %q", req.SnapshotID)})
 		return
 	}
 
 	store := providerObjectStoreFactory(cfg)
 	if err := deleteObjectPrefix(ctx, store, cfg.Bucket, req.SnapshotID+"/"); err != nil {
 		slog.Error("ebs.provider.snapshot.delete: failed", "snapshot", req.SnapshotID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("delete snapshot objects: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: internalError("delete snapshot objects: %v", err)})
 		return
 	}
 
 	slog.Info("ebs.provider.snapshot.delete: deleted", "snapshot", req.SnapshotID)
-	respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned()})
+	respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v})
 }
 
 // handleCopySnapshot serves ebs.provider.v1.snapshot.copy. Unlike
@@ -955,21 +990,23 @@ func handleDeleteSnapshot(ctx context.Context, cfg *Config, msg *nats.Msg) {
 // pattern.
 func handleCopySnapshot(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.CopySnapshotRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.snapshot.copy: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.SourceSnapshotID) || !validVolumeName(req.DestinationSnapshotID) || !validVolumeName(req.VolumeID) {
-		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid source snapshot, destination snapshot, or volume id")})
+		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: invalidArgumentError("invalid source snapshot, destination snapshot, or volume id")})
 		return
 	}
 	if req.SourceSnapshotID == req.DestinationSnapshotID {
-		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("destination snapshot %q must differ from the source", req.DestinationSnapshotID)})
+		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: invalidArgumentError("destination snapshot %q must differ from the source", req.DestinationSnapshotID)})
 		return
 	}
 
@@ -977,20 +1014,20 @@ func handleCopySnapshot(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	if err != nil {
 		switch {
 		case errors.Is(err, errSnapshotDestinationExists):
-			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: alreadyExistsError("snapshot %s already exists", req.DestinationSnapshotID)})
+			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: alreadyExistsError("snapshot %s already exists", req.DestinationSnapshotID)})
 		case errors.Is(err, viperblock.ErrSnapshotVolumeMismatch):
-			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("%v", err)})
+			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: invalidArgumentError("%v", err)})
 		case errors.Is(err, viperblock.ErrStateNotFound), errors.Is(err, os.ErrNotExist):
-			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: notFoundError("source snapshot %s not found on volume %s", req.SourceSnapshotID, req.VolumeID)})
+			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: notFoundError("source snapshot %s not found on volume %s", req.SourceSnapshotID, req.VolumeID)})
 		default:
 			slog.Error("ebs.provider.snapshot.copy: failed", "volume", req.VolumeID, "source", req.SourceSnapshotID, "destination", req.DestinationSnapshotID, "err", err)
-			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("copy snapshot: %v", err)})
+			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: internalError("copy snapshot: %v", err)})
 		}
 		return
 	}
 
 	slog.Info("ebs.provider.snapshot.copy: copied", "volume", req.VolumeID, "source", req.SourceSnapshotID, "destination", req.DestinationSnapshotID)
-	respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Snapshot: snapshot})
+	respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Snapshot: snapshot})
 }
 
 // snapshotConfigExists probes whether snapshotID's config object is already
@@ -1055,13 +1092,15 @@ func copySnapshotMetaWithVB(vb *viperblock.VB, volumeID, srcSnapshotID, dstSnaps
 
 func handleCreateSnapshot(ctx context.Context, cfg *Config, nc *nats.Conn, msg *nats.Msg) {
 	var req ebsprovider.CreateSnapshotRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.snapshot.create: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 
@@ -1076,23 +1115,23 @@ func handleCreateSnapshot(ctx context.Context, cfg *Config, nc *nats.Conn, msg *
 	}
 	if !validVolumeName(subjectVolumeID) || subjectVolumeID != req.VolumeID {
 		slog.Error("ebs.provider.snapshot.create: subject/volume mismatch", "subject", msg.Subject, "volume", req.VolumeID)
-		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("volume id %q in subject does not match request volume id %q", subjectVolumeID, req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: invalidArgumentError("volume id %q in subject does not match request volume id %q", subjectVolumeID, req.VolumeID)})
 		return
 	}
 	if !validVolumeName(req.SnapshotID) {
-		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid snapshot id %q", req.SnapshotID)})
+		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: invalidArgumentError("invalid snapshot id %q", req.SnapshotID)})
 		return
 	}
 
 	completionSubject, err := ebsprovider.SnapshotCompletionSubject(req.SnapshotID)
 	if err != nil {
-		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("%s", err.Error())})
+		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: invalidArgumentError("%s", err.Error())})
 		return
 	}
 
 	operationID := awsidentifiers.GenerateResourceID("op")
 	respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{
-		Versioned:         ebsprovider.NewVersioned(),
+		Versioned:         v,
 		OperationID:       operationID,
 		CompletionSubject: completionSubject,
 		Snapshot: &ebsprovider.Snapshot{
@@ -1117,7 +1156,7 @@ func handleCreateSnapshot(ctx context.Context, cfg *Config, nc *nats.Conn, msg *
 // contract NATSProvider.CreateSnapshot waits on.
 func completeCreateSnapshot(ctx context.Context, cfg *Config, nc *nats.Conn, req ebsprovider.CreateSnapshotRequest, operationID, completionSubject string) {
 	completion := ebsprovider.CreateSnapshotResponse{
-		Versioned:         ebsprovider.NewVersioned(),
+		Versioned:         ebsprovider.ReplyVersioned(req.SchemaVersion),
 		OperationID:       operationID,
 		CompletionSubject: completionSubject,
 	}
@@ -1396,19 +1435,20 @@ func mountVolume(ctx context.Context, cfg *Config, nc *nats.Conn, volumeName str
 	// ebs.mount subject is the route production actually takes: a retried
 	// attach racing a fresh one would otherwise start two real exports.
 	if mv, ok := findMountedVolume(cfg, volumeName); ok {
-		// An entry whose socket is gone describes an export that no longer
-		// exists, and returning its URI makes the volume unstartable forever:
-		// every caller waits for a socket nothing will create. Dropping it and
-		// mounting again is safe because an absent socket cannot be serving.
+		// Returning a dead entry's URI makes the volume unstartable forever:
+		// every caller waits for a socket nothing will create. The entry is
+		// dropped and the volume mounted again once its nbdkit is gone.
 		if stale, why := mountEntryIsStale(mv); stale {
 			slog.WarnContext(ctx, "ebs.mount: dropping stale mount entry and remounting",
 				"volume", volumeName, "uri", mv.NBDURI, "reason", why)
-			releaseStaleMount(ctx, cfg, volumeName)
+			if err := releaseStaleMount(ctx, cfg, volumeName); err != nil {
+				return viperblocklegacyv1.EBSMountResponse{Error: err.Error()}, err
+			}
 		} else {
 			// Access mode is fixed when nbdkit starts, so a remount asking for
 			// the other mode cannot be answered with the running export.
 			if mv.ReadOnly != readOnly {
-				err := fmt.Errorf("volume %s is already mounted read_only=%t on this node", volumeName, mv.ReadOnly)
+				err := fmt.Errorf("volume %s is already mounted read_only=%t on this node: %w", volumeName, mv.ReadOnly, errReadOnlyMismatch)
 				return viperblocklegacyv1.EBSMountResponse{Error: err.Error()}, err
 			}
 			slog.InfoContext(ctx, "ebs.mount: already mounted, returning existing export", "volume", volumeName, "uri", mv.NBDURI)
@@ -1675,13 +1715,13 @@ func unmountVolume(ctx context.Context, cfg *Config, volumeName string) (viperbl
 		// drains to the backend and leaves a receipt. SIGKILLing into that
 		// discards the drain and leaves the WAL for the fallback seal below to
 		// replay, which costs far more than the caller's deadline allows.
-		hostprocess.TerminateProcess(matched.PID, pluginSealGrace)
+		hostprocess.TerminateProcess(matched.PID, cfg.exportSealGrace())
 
 		// The seal below rewrites the directory nbdkit writes, so a kill that
 		// did not take makes it a concurrent write to that directory. Fail the
 		// unmount instead: the entry stays mounted and a retry re-attempts
 		// both, which is what a failed seal already does.
-		if err := hostprocess.ForceKillProcess(matched.PID, fenceKillTimeout); err != nil {
+		if err := hostprocess.ForceKillProcess(matched.PID, cfg.exportKillWait()); err != nil {
 			slog.ErrorContext(ctx, "ebs.unmount: nbdkit did not exit, refusing to seal underneath a live writer",
 				"volume", matched.Name, "pid", matched.PID, "err", err)
 			ebsResponse.Error = fmt.Sprintf("kill nbdkit for %s: %v", matched.Name, err)
@@ -1777,54 +1817,27 @@ func unmountVolume(ctx context.Context, cfg *Config, volumeName string) (viperbl
 // unlike the legacy handler there is no queue-group fallback to reason about.
 func handlePublishVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *nats.Msg) {
 	var req ebsprovider.PublishVolumeRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.publish: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
 		slog.Error("ebs.provider.volume.publish: unsupported schema version", "version", req.SchemaVersion)
-		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.VolumeID) {
-		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
 		return
 	}
-	// Idempotent republish: a volume this node already has mounted must not
-	// start a second nbdkit against it (the double-writer hazard this
-	// provider boundary exists to prevent). Match memory.go's behaviour.
-	if mv, ok := findMountedVolume(cfg, req.VolumeID); ok {
-		// Access mode is fixed when nbdkit starts, so a republish asking for
-		// the other mode cannot be answered with the running export.
-		if mv.ReadOnly != req.ReadOnly {
-			respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is already published read_only=%t on this node", req.VolumeID, mv.ReadOnly)})
-			return
-		}
-		slog.Info("ebs.provider.volume.publish: already published, returning existing attachment", "volume", req.VolumeID)
-		respondPublishedVolume(ctx, msg, req.SchemaVersion, &ebsprovider.PublishedVolume{
-			VolumeID: req.VolumeID,
-			NodeID:   cfg.NodeName,
-			Export:   ebsprovider.NewNBDExport(mv.NBDURI),
-		})
-		return
-	}
-
+	// mountVolume returns a live entry's export, and drops and remounts one
+	// whose export has died, so a republish never hands back a dead URI.
 	mountResp, err := mountVolume(ctx, cfg, nc, req.VolumeID, req.ReadOnly)
 	if err != nil {
-		if errors.Is(err, viperblock.ErrStateNotFound) {
-			respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: notFoundError("volume %s not found", req.VolumeID)})
-			return
-		}
-		// Another node holds the volume's lease. This is the exclusion working,
-		// not a failure: reporting it as internal tells the caller something
-		// broke when the answer is that someone else has the volume.
-		if errors.Is(err, errVolumeLeaseHeld) {
-			respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is held by another node: %v", req.VolumeID, err)})
-			return
-		}
-		slog.Error("ebs.provider.volume.publish: mount failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("mount volume: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: publishError(req.SchemaVersion, req.VolumeID, err)})
 		return
 	}
 
@@ -1836,59 +1849,110 @@ func handlePublishVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *n
 	})
 }
 
+// publishError classifies a failed mount for the contract reply. A lease held
+// by another node is the exclusion working, and a backing store that is not
+// ready yet is worth retrying, so neither is reported as internal.
+func publishError(version uint16, volumeID string, err error) *ebsprovider.ProviderError {
+	switch {
+	case errors.Is(err, viperblock.ErrStateNotFound):
+		return notFoundError("volume %s not found", volumeID)
+	case errors.Is(err, errVolumeLeaseHeld) && version >= ebsprovider.HeldElsewhereSchemaVersion:
+		return heldElsewhereError("volume %s is held by another node: %v", volumeID, err)
+	case errors.Is(err, errVolumeLeaseHeld):
+		// Below version 3 the refusal keeps the code v1.23.0 sends.
+		return volumeInUseError("volume %s is held by another node: %v", volumeID, err)
+	case errors.Is(err, errReadOnlyMismatch):
+		return volumeInUseError("%v", err)
+	case mountErrRetryable(err), errors.Is(err, ebsprovider.ErrUnavailable):
+		return unavailableError("mount volume %s: %v", volumeID, err)
+	default:
+		slog.Error("ebs.provider.volume.publish: mount failed", "volume", volumeID, "err", err)
+		return internalError("mount volume: %v", err)
+	}
+}
+
 // handleUnpublishVolume serves ebs.provider.v1.<node>.unmount, fronting the
 // same seal-decision path ebs.unmount uses. A failed seal is reported as an
 // error while leaving the volume in cfg.MountedVolumes (see unmountVolume),
 // never as success.
 func handleUnpublishVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.UnpublishVolumeRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.unpublish: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
-	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		slog.Error("ebs.provider.volume.unpublish: unsupported schema version", "version", req.SchemaVersion)
-		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+	if err := ebsprovider.CheckUnpublishVersion(req); err != nil {
+		slog.Error("ebs.provider.volume.unpublish: request refused", "version", req.SchemaVersion, "no_seal", req.NoSeal, "err", err)
+		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: unpublishRequestError(req.SchemaVersion, err)})
 		return
 	}
 	if !validVolumeName(req.VolumeID) {
-		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
 		return
 	}
 
+	tornDown, err := unpublishVolume(ctx, cfg, req)
+	if err != nil {
+		slog.Error("ebs.provider.volume.unpublish: unmount failed", "volume", req.VolumeID, "no_seal", req.NoSeal, "err", err)
+		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: internalError("unmount volume: %v", err)})
+		return
+	}
+
+	slog.Info("ebs.provider.volume.unpublish: unpublished", "volume", req.VolumeID, "node", cfg.NodeName, "no_seal", req.NoSeal, "torn_down", tornDown)
+	respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, TornDown: tornDown})
+}
+
+// unpublishRequestError classifies a CheckUnpublishVersion refusal.
+func unpublishRequestError(version uint16, err error) *ebsprovider.ProviderError {
+	if errors.Is(err, ebsprovider.ErrUnsupportedVersion) {
+		return unsupportedVersionError(version)
+	}
+	return invalidArgumentError("%v", err)
+}
+
+// unpublishVolume tears down this node's export and reports whether there was
+// one, which is only reported from NoSealSchemaVersion on. NoSeal drops it
+// unsealed, as an abandon does; otherwise it seals first, as ebs.unmount does.
+func unpublishVolume(ctx context.Context, cfg *Config, req ebsprovider.UnpublishVolumeRequest) (bool, error) {
+	tornDown, err := tearDownPublication(ctx, cfg, req)
+	return tornDown && req.SchemaVersion >= ebsprovider.NoSealSchemaVersion, err
+}
+
+func tearDownPublication(ctx context.Context, cfg *Config, req ebsprovider.UnpublishVolumeRequest) (bool, error) {
+	if req.NoSeal {
+		return cfg.abandonVolume(ctx, req.VolumeID, "unpublish with no_seal")
+	}
 	unmountResp, err := unmountVolume(ctx, cfg, req.VolumeID)
 	if err != nil {
 		// Nothing mounted here is already the state unpublish asks for, so a
 		// retry after a request that completed server-side converges rather
 		// than erroring, matching the legacy path's unmountResponseError.
 		if unmountResp.NotFound {
-			slog.Info("ebs.provider.volume.unpublish: already unpublished", "volume", req.VolumeID, "node", cfg.NodeName)
-			respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned()})
-			return
+			return unmountResp.Reaped, nil
 		}
-		slog.Error("ebs.provider.volume.unpublish: unmount failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("unmount volume: %v", err)})
-		return
+		return false, err
 	}
-
-	slog.Info("ebs.provider.volume.unpublish: unpublished", "volume", req.VolumeID, "node", cfg.NodeName)
-	respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned()})
+	return true, nil
 }
 
 func handleHoldsUnsealedWrites(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.HoldsUnsealedWritesRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
-		respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
+		respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	for _, volume := range req.VolumeIDs {
 		if !validVolumeName(volume) {
-			respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", volume)})
+			respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", volume)})
 			return
 		}
 	}
@@ -1896,19 +1960,18 @@ func handleHoldsUnsealedWrites(ctx context.Context, cfg *Config, msg *nats.Msg)
 	held, err := cfg.unsealedWritesHeld(ctx, req.VolumeIDs)
 	if err != nil {
 		slog.Error("ebs.provider.unsealed: could not read dirty markers", "node", cfg.NodeName, "err", err)
-		respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("read dirty markers: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: internalError("read dirty markers: %v", err)})
 		return
 	}
-	respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: ebsprovider.NewVersioned(), Held: held})
+	respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Held: held})
 }
 
-// mountEntryIsStale reports whether a MountedVolume describes an export that is
-// no longer running, and why.
+// mountEntryIsStale reports whether a MountedVolume describes an export that no
+// new client can reach, and why.
 //
-// Only positive evidence of death counts. The entry guards against a second
-// nbdkit for one volume, so treating a live export as stale would create the
-// double-writer hazard it exists to prevent — an absent socket or a dead pid
-// are the two conditions under which no export can be serving.
+// Only positive evidence counts: an absent socket or a dead pid. An absent
+// socket does not mean the nbdkit has exited, since a client that connected
+// before it went keeps writing through it, so releaseStaleMount stops it.
 func mountEntryIsStale(mv MountedVolume) (bool, string) {
 	if mv.Socket != "" {
 		if _, err := os.Stat(mv.Socket); errors.Is(err, os.ErrNotExist) {
@@ -1923,7 +1986,18 @@ func mountEntryIsStale(mv MountedVolume) (bool, string) {
 
 // releaseStaleMount drops a dead export's entry and everything it held, so the
 // remount that follows starts from the same state a clean unmount would leave.
-func releaseStaleMount(ctx context.Context, cfg *Config, volumeName string) {
+//
+// An nbdkit still running behind a lost socket is a writer, so it goes through
+// unmountVolume, which stops it and seals its copy. If it will not stop, the
+// entry stays and the error wraps ErrUnavailable: a remount would be a second writer.
+func releaseStaleMount(ctx context.Context, cfg *Config, volumeName string) error {
+	if mv, ok := findMountedVolume(cfg, volumeName); ok && hostprocess.ProcessAlive(mv.PID) {
+		if _, err := unmountVolume(ctx, cfg, volumeName); err != nil {
+			return fmt.Errorf("%w: stop nbdkit %d behind the lost socket of %s: %w", ebsprovider.ErrUnavailable, mv.PID, volumeName, err)
+		}
+		return nil
+	}
+
 	cfg.mu.Lock()
 	var matched MountedVolume
 	found := false
@@ -1936,7 +2010,7 @@ func releaseStaleMount(ctx context.Context, cfg *Config, volumeName string) {
 	}
 	cfg.mu.Unlock()
 	if !found {
-		return
+		return nil
 	}
 
 	if matched.ConfigSub != nil {
@@ -1961,4 +2035,5 @@ func releaseStaleMount(ctx context.Context, cfg *Config, volumeName string) {
 		}
 	}
 	cfg.releaseVolumeLease(ctx, matched.Lease)
+	return nil
 }