feat(lambda): add ListVersionsByFunction listing only $LATEST

Until versions can be published, the only version is $LATEST: the list holds
its configuration with the FunctionArn qualified as :$LATEST and no
NextMarker. MaxItems outside 1 to 10000 is refused, and Marker is refused
outright rather than guessing AWS's paging behaviour. It is authorized as
lambda:ListVersionsByFunction on the function ARN and does not need the code
store.

cmd/lambda-agent/package_testlink.go

@@ -0,0 +1,276 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// Dev-only code delivery for the test path: the host attaches the ZIP as a
+// read-only disk and passes its exact size and SHA-256 through fw_cfg. This
+// is not the production delivery, and these bounds are not AWS limits.
+const (
+	entryPackageSize   = "test-package-size"
+	entryPackageSHA256 = "test-package-sha256"
+
+	devMaxZipBytes    = 16 << 20
+	devMaxUnzipBytes  = 64 << 20
+	devMaxEntries     = 1024
+	packageDevice     = "/dev/vda"
+	taskTmpfsSizeOpts = "mode=0755,size=72m"
+)
+
+type packageSpec struct {
+	size   int64
+	sha256 [sha256.Size]byte
+}
+
+type unpackLimits struct {
+	entries int
+	bytes   int64
+}
+
+var devLimits = unpackLimits{entries: devMaxEntries, bytes: devMaxUnzipBytes}
+
+// readPackageSpec reads the expected size and SHA-256 from fw_cfg; it must
+// run before fw_cfg is sealed.
+func readPackageSpec(fwCfgDir string) (packageSpec, error) {
+	var spec packageSpec
+	read := func(name string) (string, error) {
+		b, err := os.ReadFile(filepath.Join(fwCfgDir, name, "raw"))
+		if err != nil {
+			return "", fmt.Errorf("read %s: %w", name, err)
+		}
+		return strings.TrimSpace(string(b)), nil
+	}
+	s, err := read(entryPackageSize)
+	if err != nil {
+		return spec, err
+	}
+	spec.size, err = strconv.ParseInt(s, 10, 64)
+	if err != nil || spec.size <= 0 || spec.size > devMaxZipBytes {
+		return spec, fmt.Errorf("package size %q outside 1..%d bytes", s, devMaxZipBytes)
+	}
+	h, err := read(entryPackageSHA256)
+	if err != nil {
+		return spec, err
+	}
+	b, err := hex.DecodeString(h)
+	if err != nil || len(b) != sha256.Size {
+		return spec, errors.New("package sha256 is not 64 hex digits")
+	}
+	copy(spec.sha256[:], b)
+	return spec, nil
+}
+
+// taskPackage puts the package under dir: a fresh nosuid,nodev tmpfs, made
+// read-only once the files are in place.
+type taskPackage struct {
+	device     string
+	dir        string
+	loadDriver func() error
+	mountDir   func(dir string) error
+	sealDir    func(dir string) error
+}
+
+func devTaskPackage() taskPackage {
+	return taskPackage{device: packageDevice, dir: taskRoot, loadDriver: loadBlockDriver, mountDir: mountTaskDir, sealDir: sealTaskDir}
+}
+
+func (p taskPackage) install(spec packageSpec) error {
+	if err := p.loadDriver(); err != nil {
+		return fmt.Errorf("load block driver: %w", err)
+	}
+	dev, err := os.Open(p.device)
+	if err != nil {
+		return fmt.Errorf("open package disk: %w", err)
+	}
+	defer dev.Close()
+	if err := p.mountDir(p.dir); err != nil {
+		return fmt.Errorf("mount %s: %w", p.dir, err)
+	}
+	if err := unpack(dev, spec, p.dir, devLimits); err != nil {
+		return err
+	}
+	if err := p.sealDir(p.dir); err != nil {
+		return fmt.Errorf("make %s read-only: %w", p.dir, err)
+	}
+	return nil
+}
+
+// unpack reads exactly spec.size bytes once, checks their SHA-256, then
+// validates every entry before writing any of them under dir.
+func unpack(r io.ReaderAt, spec packageSpec, dir string, lim unpackLimits) error {
+	buf := make([]byte, spec.size)
+	if _, err := io.ReadFull(io.NewSectionReader(r, 0, spec.size), buf); err != nil {
+		return fmt.Errorf("read package: %w", err)
+	}
+	if sha256.Sum256(buf) != spec.sha256 {
+		return errors.New("package sha256 mismatch")
+	}
+	zr, err := zip.NewReader(bytes.NewReader(buf), spec.size)
+	if err != nil {
+		return fmt.Errorf("open package: %w", err)
+	}
+	if len(zr.File) > lim.entries {
+		return fmt.Errorf("package has %d entries, limit %d", len(zr.File), lim.entries)
+	}
+	names := make([]string, len(zr.File))
+	var declared int64
+	for i, f := range zr.File {
+		name, err := entryName(f.Name)
+		if err != nil {
+			return err
+		}
+		names[i] = name
+		switch mode := f.Mode(); {
+		case mode.IsDir():
+		case mode.IsRegular():
+			// Compared against what remains, so a huge size cannot wrap the sum.
+			if f.UncompressedSize64 > uint64(lim.bytes-declared) {
+				return fmt.Errorf("%q: unpacked size over the %d-byte limit", f.Name, lim.bytes)
+			}
+			declared += int64(f.UncompressedSize64)
+		default:
+			return fmt.Errorf("%q: %s entries are refused", f.Name, mode.Type())
+		}
+	}
+
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		return err
+	}
+	defer root.Close()
+	var written int64
+	for i, f := range zr.File {
+		n, err := extract(root, names[i], f, lim.bytes-written)
+		written += n
+		if err != nil {
+			return fmt.Errorf("%q: %w", f.Name, err)
+		}
+	}
+	return nil
+}
+
+// extract writes one entry, refusing it if its actual bytes exceed budget
+// whatever its header claims.
+func extract(root *os.Root, name string, f *zip.File, budget int64) (int64, error) {
+	if f.Mode().IsDir() {
+		return 0, mkdirs(root, name)
+	}
+	if d := path.Dir(name); d != "." {
+		if err := mkdirs(root, d); err != nil {
+			return 0, err
+		}
+	}
+	perm := os.FileMode(0o644)
+	if f.Mode()&0o111 != 0 {
+		perm = 0o755
+	}
+	in, err := f.Open()
+	if err != nil {
+		return 0, err
+	}
+	defer in.Close()
+	out, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
+	if err != nil {
+		return 0, err
+	}
+	// The umask applies at create; fchmod does not, so the mode is exact.
+	if err := out.Chmod(perm); err != nil {
+		out.Close()
+		return 0, err
+	}
+	n, err := io.Copy(out, io.LimitReader(in, budget+1))
+	if cerr := out.Close(); err == nil {
+		err = cerr
+	}
+	if err == nil && n > budget {
+		err = errors.New("unpacked bytes over the limit")
+	}
+	return n, err
+}
+
+// mkdirs creates each component of dir as mode 0755 whatever the umask. An
+// existing component must be a real directory, never a symlink.
+func mkdirs(root *os.Root, dir string) error {
+	cur := ""
+	for _, part := range strings.Split(dir, "/") {
+		cur = path.Join(cur, part)
+		if err := root.Mkdir(cur, 0o755); err != nil {
+			if !errors.Is(err, fs.ErrExist) {
+				return err
+			}
+			fi, err := root.Lstat(cur)
+			if err != nil {
+				return err
+			}
+			if !fi.IsDir() {
+				return fmt.Errorf("%q exists and is not a directory", cur)
+			}
+		}
+		if err := root.Chmod(cur, 0o755); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// entryName returns a clean relative path, refusing anything that could name
+// a location outside the task root.
+func entryName(name string) (string, error) {
+	if name == "" || strings.ContainsAny(name, "\\\x00") || path.IsAbs(name) {
+		return "", fmt.Errorf("%q: entry name refused", name)
+	}
+	c := path.Clean(name)
+	if c == "." || c == ".." || strings.HasPrefix(c, "../") {
+		return "", fmt.Errorf("%q: entry name refused", name)
+	}
+	return c, nil
+}
+
+// loadBlockDriver loads virtio_blk, which the image ships but init does not
+// load, then waits briefly for the package disk to appear.
+func loadBlockDriver() error {
+	if err := exec.Command("/sbin/modprobe", "virtio_blk").Run(); err != nil {
+		var uts unix.Utsname
+		if uerr := unix.Uname(&uts); uerr != nil {
+			return uerr
+		}
+		ko := "/lib/modules/" + unix.ByteSliceToString(uts.Release[:]) + "/kernel/drivers/block/virtio_blk.ko"
+		if err := exec.Command("/sbin/insmod", ko).Run(); err != nil {
+			return fmt.Errorf("modprobe and insmod virtio_blk failed: %w", err)
+		}
+	}
+	for range 50 {
+		if _, err := os.Stat(packageDevice); err == nil {
+			return nil
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	return fmt.Errorf("%s did not appear", packageDevice)
+}
+
+func mountTaskDir(dir string) error {
+	return unix.Mount("tmpfs", dir, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, taskTmpfsSizeOpts)
+}
+
+func sealTaskDir(dir string) error {
+	return unix.Mount("", dir, "", unix.MS_REMOUNT|unix.MS_RDONLY|unix.MS_NOSUID|unix.MS_NODEV, "")
+}

cmd/lambda-agent/package_testlink_test.go

@@ -0,0 +1,209 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"hash/crc32"
+	"io/fs"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testZip builds a ZIP; an entry named bootstrap is marked executable.
+func testZip(t *testing.T, files map[string]string) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	names := make([]string, 0, len(files))
+	for n := range files {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	for _, n := range names {
+		h := &zip.FileHeader{Name: n, Method: zip.Deflate}
+		h.SetMode(0o644)
+		if n == "bootstrap" {
+			h.SetMode(0o755)
+		}
+		w, err := zw.CreateHeader(h)
+		require.NoError(t, err)
+		_, err = w.Write([]byte(files[n]))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+// rawZip writes entries with headers set by the caller, so a test can claim
+// sizes or modes the data does not have.
+func rawZip(t *testing.T, entries ...*zip.FileHeader) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for _, h := range entries {
+		w, err := zw.CreateRaw(h)
+		require.NoError(t, err)
+		_, err = w.Write([]byte("x"))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+func specFor(b []byte) packageSpec {
+	return packageSpec{size: int64(len(b)), sha256: sha256.Sum256(b)}
+}
+
+// Not parallel: the umask is process-wide, so the test sets a restrictive one
+// and restores it.
+func TestUnpackInstallsFilesWithFixedModes(t *testing.T) {
+	old := syscall.Umask(0o077)
+	t.Cleanup(func() { syscall.Umask(old) })
+	b := testZip(t, map[string]string{"bootstrap": "#!/bin/sh\n", "lib/data.txt": "data"})
+	dir := t.TempDir()
+	require.NoError(t, unpack(bytes.NewReader(b), specFor(b), dir, devLimits))
+
+	pfi, err := os.Stat(filepath.Join(dir, "lib"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o755), pfi.Mode().Perm(), "implicit parent dir")
+
+	fi, err := os.Stat(filepath.Join(dir, "bootstrap"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o755), fi.Mode().Perm(), "executable")
+	fi, err = os.Stat(filepath.Join(dir, "lib", "data.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o644), fi.Mode().Perm())
+	got, err := os.ReadFile(filepath.Join(dir, "lib", "data.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, "data", string(got))
+}
+
+func TestUnpackRefusesBeforeWritingAnything(t *testing.T) {
+	symlink := &zip.FileHeader{Name: "link", Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: 1}
+	symlink.SetMode(fs.ModeSymlink | 0o777)
+	fifo := &zip.FileHeader{Name: "pipe", Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: 1}
+	fifo.SetMode(fs.ModeNamedPipe | 0o644)
+	// The second size would wrap a plain running sum back under the limit.
+	big := func(name string, n uint64) *zip.FileHeader {
+		return &zip.FileHeader{Name: name, Method: zip.Store, CRC32: crc32.ChecksumIEEE([]byte("x")), CompressedSize64: 1, UncompressedSize64: n}
+	}
+	cases := map[string]struct {
+		zip  []byte
+		want string
+	}{
+		"parent traversal": {testZip(t, map[string]string{"a": "ok", "../escape": "x"}), "entry name refused"},
+		"nested traversal": {testZip(t, map[string]string{"a": "ok", "lib/../../escape": "x"}), "entry name refused"},
+		"absolute path":    {testZip(t, map[string]string{"a": "ok", "/etc/escape": "x"}), "entry name refused"},
+		"backslash":        {testZip(t, map[string]string{"a": "ok", `..\escape`: "x"}), "entry name refused"},
+		"symlink":          {rawZip(t, big("a", 1), symlink), "entries are refused"},
+		"named pipe":       {rawZip(t, big("a", 1), fifo), "entries are refused"},
+		"declared size":    {rawZip(t, big("a", devMaxUnzipBytes+1)), "over the"},
+		"wrapping sizes":   {rawZip(t, big("a", devMaxUnzipBytes), big("b", math.MaxUint64)), "over the"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := t.TempDir()
+			err := unpack(bytes.NewReader(tc.zip), specFor(tc.zip), dir, devLimits)
+			assert.ErrorContains(t, err, tc.want)
+			left, rerr := os.ReadDir(dir)
+			require.NoError(t, rerr)
+			assert.Empty(t, left, "files written before the refusal")
+		})
+	}
+}
+
+func TestUnpackRefusesTooManyEntries(t *testing.T) {
+	b := testZip(t, map[string]string{"a": "1", "b": "2", "c": "3"})
+	err := unpack(bytes.NewReader(b), specFor(b), t.TempDir(), unpackLimits{entries: 2, bytes: devMaxUnzipBytes})
+	assert.ErrorContains(t, err, "limit 2")
+}
+
+// The header claims a size within the limit, but the data is longer: the
+// actual bytes are refused, whatever the header says.
+func TestUnpackRefusesBytesBeyondTheLimit(t *testing.T) {
+	data := bytes.Repeat([]byte("a"), 64)
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	h := &zip.FileHeader{Name: "a", Method: zip.Store, CRC32: crc32.ChecksumIEEE(data), CompressedSize64: uint64(len(data)), UncompressedSize64: 8}
+	w, err := zw.CreateRaw(h)
+	require.NoError(t, err)
+	_, err = w.Write(data)
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	b := buf.Bytes()
+
+	dir := t.TempDir()
+	err = unpack(bytes.NewReader(b), specFor(b), dir, unpackLimits{entries: 4, bytes: 16})
+	require.Error(t, err)
+	fi, serr := os.Stat(filepath.Join(dir, "a"))
+	if serr == nil {
+		assert.LessOrEqual(t, fi.Size(), int64(17))
+	}
+}
+
+func TestUnpackChecksSizeAndHash(t *testing.T) {
+	b := testZip(t, map[string]string{"bootstrap": "x"})
+	wrongHash := specFor(b)
+	wrongHash.sha256[0] ^= 1
+	assert.ErrorContains(t, unpack(bytes.NewReader(b), wrongHash, t.TempDir(), devLimits), "sha256 mismatch")
+
+	longer := specFor(b)
+	longer.size++
+	assert.ErrorContains(t, unpack(bytes.NewReader(b), longer, t.TempDir(), devLimits), "read package")
+}
+
+func TestReadPackageSpecBounds(t *testing.T) {
+	sum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	cases := map[string]struct {
+		size, hash string
+		want       string
+	}{
+		"valid":     {"10", sum, ""},
+		"zero":      {"0", sum, "outside"},
+		"too large": {"16777217", sum, "outside"},
+		"not hex":   {"10", "zz", "64 hex digits"},
+		"short":     {"10", sum[:62], "64 hex digits"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := t.TempDir()
+			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash} {
+				require.NoError(t, os.MkdirAll(filepath.Join(dir, n), 0o700))
+				require.NoError(t, os.WriteFile(filepath.Join(dir, n, "raw"), []byte(v), 0o600))
+			}
+			spec, err := readPackageSpec(dir)
+			if tc.want == "" {
+				require.NoError(t, err)
+				assert.Equal(t, int64(10), spec.size)
+				return
+			}
+			assert.ErrorContains(t, err, tc.want)
+		})
+	}
+}
+
+// A pre-existing symlink where a directory belongs is refused, not followed
+// or chmodded.
+func TestMkdirsRefusesSymlinkComponent(t *testing.T) {
+	dir := t.TempDir()
+	outside := t.TempDir()
+	require.NoError(t, os.Chmod(outside, 0o700))
+	require.NoError(t, os.Symlink(outside, filepath.Join(dir, "lib")))
+	root, err := os.OpenRoot(dir)
+	require.NoError(t, err)
+	defer root.Close()
+	assert.ErrorContains(t, mkdirs(root, "lib/sub"), "not a directory")
+	fi, err := os.Stat(outside)
+	require.NoError(t, err)
+	assert.Equal(t, fs.FileMode(0o700), fi.Mode().Perm(), "symlink target was chmodded")
+}

cmd/lambda-agent/start_testlink.go

@@ -30,6 +30,7 @@ var handler = flag.String("handler", os.Getenv("_HANDLER"), "function handler pa
 // connection; the carrier is not the guest protocol.
 type testlink struct {
 	startup
+	pkg            taskPackage
 	runtimeAddr    string
 	startBootstrap func(ctx context.Context, env []string) (*exec.Cmd, error)
 }
@@ -37,6 +38,7 @@ type testlink struct {
 func start(ctx context.Context) int {
 	t := testlink{
 		startup:     defaultStartup(),
+		pkg:         devTaskPackage(),
 		runtimeAddr: runtimeAddr,
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			return agent.StartBootstrap(ctx, taskRoot+"/bootstrap", env, os.Stdout, os.Stderr)
@@ -50,11 +52,16 @@ func start(ctx context.Context) int {
 }
 
 func (t testlink) run(parent context.Context) (int, error) {
+	// The package values are fw_cfg entries, so read them before connect seals it.
+	spec, specErr := readPackageSpec(t.fwCfgDir)
 	conn, code, err := t.connect(parent)
 	if err != nil {
-		return code, err
+		return code, errors.Join(err, specErr)
 	}
 	defer conn.Close()
+	if specErr != nil {
+		return exitRefused, specErr
+	}
 	link := hostlink.NewTestStream(conn)
 
 	// The handshake can return before the host refuses the guest, so nothing
@@ -69,6 +76,9 @@ func (t testlink) run(parent context.Context) (int, error) {
 	if parent.Err() != nil {
 		return exitLink, context.Cause(parent)
 	}
+	if err := t.pkg.install(spec); err != nil {
+		return exitRefused, fmt.Errorf("install package: %w", err)
+	}
 
 	srv := runtimeapi.New()
 	ln, err := net.Listen("tcp", t.runtimeAddr)

cmd/lambda-agent/start_testlink_test.go

@@ -5,10 +5,14 @@ package main
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"io"
 	"net/http"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"strconv"
 	"testing"
 	"time"
 
@@ -43,12 +47,31 @@ func TestHelperBootstrap(t *testing.T) {
 type bootstrapStarts struct {
 	n       int
 	started chan struct{}
+	task    string
+	sealed  bool
 }
 
-func (b *bootstrapStarts) testlink(g *guest) testlink {
+// testlink builds a guest whose fw_cfg also carries a controlled package,
+// installed into a temp dir without mounting anything.
+func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) testlink {
+	t.Helper()
+	zipBytes := testZip(t, map[string]string{"bootstrap": "#!/bin/sh\n", "lib/data.txt": "data"})
+	disk := filepath.Join(t.TempDir(), "disk")
+	require.NoError(t, os.WriteFile(disk, zipBytes, 0o600))
+	sum := sha256.Sum256(zipBytes)
+	entries[entryPackageSize] = []byte(strconv.Itoa(len(zipBytes)))
+	entries[entryPackageSHA256] = []byte(hex.EncodeToString(sum[:]))
+	g := newGuest(t, entries)
 	b.started = make(chan struct{}, 1)
+	b.task = t.TempDir()
 	return testlink{
-		startup:     g.startup(),
+		startup: g.startup(),
+		pkg: taskPackage{
+			device: disk, dir: b.task,
+			loadDriver: func() error { return nil },
+			mountDir:   func(string) error { return nil },
+			sealDir:    func(string) error { b.sealed = true; return nil },
+		},
 		runtimeAddr: "127.0.0.1:0",
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			b.n++
@@ -80,7 +103,7 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
 
 	conn := h.guest(t)
 	defer conn.Close()
@@ -109,6 +132,8 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		assert.Error(t, err, "agent answered a second invocation")
 	}
 	assert.Equal(t, 1, b.n)
+	assert.True(t, b.sealed, "task dir not made read-only")
+	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
 }
 
 func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
@@ -117,7 +142,7 @@ func TestTestlinkShutdownBeforeInvocationNeverStartsBootstrap(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
 
 	conn := h.guest(t)
 	defer conn.Close()
@@ -142,7 +167,7 @@ func TestTestlinkHostRefusalNeverStartsBootstrap(t *testing.T) {
 	entries[lambdav1.EntryGuestKey], entries[lambdav1.EntryGuestCert] = wrong[lambdav1.EntryGuestKey], wrong[lambdav1.EntryGuestCert]
 	var b bootstrapStarts
 
-	res := wait(t, runTestlinkAsync(ctx, b.testlink(newGuest(t, entries))), "agent to stop")
+	res := wait(t, runTestlinkAsync(ctx, b.testlink(t, entries)), "agent to stop")
 	assert.Equal(t, exitLink, res.code)
 	require.Error(t, res.err)
 	assert.Contains(t, res.err.Error(), "bad certificate")
@@ -155,7 +180,7 @@ func TestTestlinkHostCloseBeforeFrameNeverStartsBootstrap(t *testing.T) {
 	h := newHost(t)
 	h.accept(ctx)
 	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(newGuest(t, h.entries())))
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
 
 	require.NoError(t, h.guest(t).Close())
 	res := wait(t, done, "agent to stop")
@@ -177,3 +202,45 @@ func TestOneInvocationHonoursCancelledContext(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "r-1", req.RequestID)
 }
+
+// A package that fails its check is refused after authentication, and the
+// bootstrap never starts.
+func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	tl := b.testlink(t, h.entries())
+	tl.pkg.device = filepath.Join(t.TempDir(), "other")
+	require.NoError(t, os.WriteFile(tl.pkg.device, testZip(t, map[string]string{"bootstrap": "x"}), 0o600))
+	done := runTestlinkAsync(ctx, tl)
+
+	conn := h.guest(t)
+	defer conn.Close()
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{RequestID: "r-1"}))
+	res := wait(t, done, "agent to stop")
+	assert.Equal(t, exitRefused, res.code)
+	assert.ErrorContains(t, res.err, "install package")
+	assert.Zero(t, b.n)
+	assert.False(t, b.sealed)
+}
+
+// Missing package values still seal fw_cfg, and the agent refuses before
+// waiting for any frame.
+func TestTestlinkMissingPackageSpecRefuses(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	tl := b.testlink(t, h.entries())
+	require.NoError(t, os.RemoveAll(filepath.Join(tl.fwCfgDir, entryPackageSHA256)))
+
+	res := wait(t, runTestlinkAsync(ctx, tl), "agent to stop")
+	assert.Equal(t, exitRefused, res.code)
+	assert.ErrorContains(t, res.err, "read "+entryPackageSHA256)
+	assert.Zero(t, b.n)
+	_, err := os.Stat(tl.fwCfgRoot)
+	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg not sealed")
+}

contracts/operator/v1/admin.go

@@ -25,6 +25,10 @@ const (
 	DeletionStateRunning   = "RUNNING"
 	DeletionStateCompleted = "COMPLETED"
 	DeletionStateFailed    = "FAILED"
+
+	// DeletionStateExercised ends a non-finalizing run: the account is left
+	// TERMINATING and a later DeleteAccount takes the same operation over.
+	DeletionStateExercised = "EXERCISED"
 )
 
 // CreateAccountRequest is the /admin/CreateAccount request body.
@@ -52,6 +56,10 @@ type DeleteAccountRequest struct {
 	ClientToken string `json:"clientToken"`
 	Force       bool   `json:"force,omitempty"`
 	DryRun      bool   `json:"dryRun,omitempty"`
+
+	// NonFinalizing quiesces the account, cleans the quiesced EC2 platform
+	// kinds and stops. No legacy stage runs and the account record stays.
+	NonFinalizing bool `json:"nonFinalizing,omitempty"`
 }
 
 // DescribeAccountDeletionRequest is the /admin/DescribeAccountDeletion body.
@@ -76,8 +84,8 @@ type ListAccountsResponse struct {
 }
 
 // IAMQuiescence is the coordinator's recorded view of IAM quiescence for one
-// fence. Recording only: it admits nothing, and QuiescedScope stays empty until
-// a final inventory re-read exists to catch a late write.
+// fence. Recording only: it admits nothing. QuiescedScope is filled only for a
+// non-finalizing run, and holds as of EvaluatedAt: a write after that is not caught.
 type IAMQuiescence struct {
 	OperationID   string            `json:"operationId"`
 	Fence         uint64            `json:"fence"`
@@ -136,3 +144,67 @@ const (
 	ReasonOwnerInventoryIncomplete     = "owner-inventory-incomplete"
 	ReasonFinalInventoryRereadNotWired = "final-inventory-reread-not-wired"
 )
+
+// ExerciseResult is what a non-finalizing run did after IAM quiescence.
+type ExerciseResult struct {
+	// Skipped is the fixed reason no cleanup ran; Cleanup is then absent.
+	Skipped    string           `json:"skipped,omitempty"`
+	Cleanup    *PlatformCleanup `json:"cleanup,omitempty"`
+	Accounting []KindAccounting `json:"accounting,omitempty"`
+}
+
+// PlatformCleanup is the EC2 platform cleanup result. It carries resource IDs,
+// fixed reasons and counts only, never names, tags, key material or data.
+type PlatformCleanup struct {
+	Kinds  []CleanupKind  `json:"kinds"`
+	Items  []CleanupItem  `json:"items,omitempty"`
+	Counts []CleanupCount `json:"counts,omitempty"`
+}
+
+// CleanupKind is one kind's scan status after the cleanup's own re-scan.
+type CleanupKind struct {
+	Kind              string   `json:"kind"`
+	Status            string   `json:"status"`
+	IncompleteReasons []string `json:"incompleteReasons,omitempty"`
+}
+
+// CleanupItem is one resource by its ID with what the cleanup did to it.
+type CleanupItem struct {
+	ResourceID  string `json:"resourceId"`
+	Kind        string `json:"kind"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	MemberCount int    `json:"memberCount,omitempty"`
+}
+
+// CleanupCount is the number of records of one class that have no public ID.
+type CleanupCount struct {
+	Kind        string `json:"kind"`
+	Class       string `json:"class"`
+	Disposition string `json:"disposition"`
+	Reason      string `json:"reason,omitempty"`
+	Count       int    `json:"count"`
+}
+
+// KindAccounting is the final inventory re-read held against the cleanup result
+// for one kind. Quiescence is not rewritten from it.
+type KindAccounting struct {
+	Kind      string `json:"kind"`
+	Status    string `json:"status"`
+	Reason    string `json:"reason,omitempty"`
+	Remaining int    `json:"remaining"`
+}
+
+// Accounting statuses and fixed reasons of a KindAccounting.
+const (
+	AccountingAccounted   = "accounted"
+	AccountingUnaccounted = "unaccounted"
+
+	AccountingReasonRereadIncomplete = "reread-incomplete"
+	AccountingReasonUnnamedRemainder = "unnamed-remainder"
+)
+
+// Fixed reasons an exercise ran no cleanup.
+const (
+	ExerciseSkippedNoQuiescedKind = "no-quiesced-kind"
+)

contracts/operator/v1/admin_test.go

@@ -43,6 +43,27 @@ func TestAdminWire_DeletionStates(t *testing.T) {
 	require.Equal(t, "RUNNING", DeletionStateRunning)
 	require.Equal(t, "COMPLETED", DeletionStateCompleted)
 	require.Equal(t, "FAILED", DeletionStateFailed)
+	require.Equal(t, "EXERCISED", DeletionStateExercised)
+}
+
+func TestAdminWire_NonFinalizingDeleteRequest(t *testing.T) {
+	goldenJSON(t, DeleteAccountRequest{AccountID: "000000000042", AccountName: "acme", ClientToken: "tok-3", NonFinalizing: true},
+		`{"accountId":"000000000042","accountName":"acme","clientToken":"tok-3","nonFinalizing":true}`)
+}
+
+func TestAdminWire_ExerciseResult(t *testing.T) {
+	goldenJSON(t, ExerciseResult{
+		Cleanup: &PlatformCleanup{
+			Kinds:  []CleanupKind{{Kind: "key-pair", Status: "observed-clean-at-scan-time"}},
+			Items:  []CleanupItem{{ResourceID: "pg-1", Kind: "placement-group", Disposition: "blocked", Reason: "instance-members", MemberCount: 1}},
+			Counts: []CleanupCount{{Kind: "key-pair", Class: "public-key-object-without-metadata", Disposition: "deleted", Count: 2}},
+		},
+		Accounting: []KindAccounting{{Kind: "placement-group", Status: AccountingAccounted, Remaining: 1}},
+	}, `{"cleanup":{"kinds":[{"kind":"key-pair","status":"observed-clean-at-scan-time"}],
+		"items":[{"resourceId":"pg-1","kind":"placement-group","disposition":"blocked","reason":"instance-members","memberCount":1}],
+		"counts":[{"kind":"key-pair","class":"public-key-object-without-metadata","disposition":"deleted","count":2}]},
+		"accounting":[{"kind":"placement-group","status":"accounted","remaining":1}]}`)
+	goldenJSON(t, ExerciseResult{Skipped: ExerciseSkippedNoQuiescedKind}, `{"skipped":"no-quiesced-kind"}`)
 }
 
 func TestAdminWire_ListAccountsResponse(t *testing.T) {

docs/DESIGN.md

@@ -65,9 +65,10 @@ The EC2 registration (`spinifex/domains/ec2/awsapi`, `NewRegistration`) parses t
 The gateway communicates with daemons via NATS request/response. Most calls go through `natsmsg.NATSRequest`, which marshals the input, attaches the account ID as a NATS header, and unmarshals the typed response:
 
 ```go
-func (s *NATSInstanceService) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
-    topic := fmt.Sprintf("ec2.RunInstances.%s", aws.StringValue(input.InstanceType))
-    return natsmsg.NATSRequest[ec2.Reservation](ctx, s.natsConn, topic, input, 5*time.Minute, accountID)
+func runIntoReservation(ctx context.Context, input *ec2.RunInstancesInput, natsConn *nats.Conn, accountID, crID string) (*ec2.Reservation, error) {
+    subject := "ec2.RunInstances.cr." + crID
+    reservation, err := natsmsg.NATSRequest[ec2.Reservation](ctx, natsConn, subject, input, 5*time.Minute, accountID)
+    // ...
 }
 ```
 

spinifex/agents/lambda/agent/bootstrap.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os/exec"
+	"path/filepath"
 	goruntime "runtime"
 	"syscall"
 	"time"
@@ -18,8 +19,8 @@ import (
 const sandboxID = 993
 
 // StartBootstrap starts the function's bootstrap at path as the sandbox uid and
-// gid with no supplementary groups, with no_new_privs, and with exactly env.
-// The caller waits on the returned command.
+// gid with no supplementary groups, with no_new_privs, and with exactly env,
+// in the bootstrap's own directory. The caller waits on the returned command.
 func StartBootstrap(ctx context.Context, path string, env []string, stdout, stderr io.Writer) (*exec.Cmd, error) {
 	cred := &syscall.Credential{Uid: sandboxID, Gid: sandboxID, Groups: []uint32{}}
 	return startBootstrap(ctx, path, env, stdout, stderr, cred)
@@ -30,6 +31,8 @@ func startBootstrap(ctx context.Context, path string, env []string, stdout, stde
 	// A nil Env inherits the agent's environment; the bootstrap gets only env.
 	cmd.Env = append([]string{}, env...)
 	cmd.Stdout, cmd.Stderr = stdout, stderr
+	// Packaged functions open their files relative to the task root.
+	cmd.Dir = filepath.Dir(path)
 	if cred != nil {
 		cmd.SysProcAttr = &syscall.SysProcAttr{Credential: cred}
 	}

spinifex/agents/lambda/agent/bootstrap_test.go

@@ -169,3 +169,11 @@ func TestStartBootstrapMissingExecutableFails(t *testing.T) {
 	_, err := startBootstrap(context.Background(), filepath.Join(t.TempDir(), "bootstrap"), nil, nil, nil, nil)
 	assert.Error(t, err)
 }
+
+func TestStartBootstrapRunsInItsOwnDirectory(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "data.txt"), []byte("packaged"), 0o644))
+	bootstrap := filepath.Join(dir, "bootstrap")
+	require.NoError(t, os.WriteFile(bootstrap, []byte("#!/bin/sh\ncat data.txt\n"), 0o755))
+	assert.Equal(t, "packaged", string(run(t, bootstrap, []string{"PATH=/bin:/usr/bin"}, false)))
+}

spinifex/bootstrap/config/config.go

@@ -5,6 +5,7 @@ package config
 import (
 	"fmt"
 	"log/slog"
+	"maps"
 	"net/netip"
 	"os"
 	"path/filepath"
@@ -355,6 +356,11 @@ type EBSConfig struct {
 	CacheSizeMB *int `json:"CacheSizeMB" mapstructure:"cache_size_mb"`
 }
 
+// SupportedEBSProvider reports whether name is a provider a node may run.
+func SupportedEBSProvider(name string) bool {
+	return name == EBSProviderViperblockd || name == EBSProviderQEMUNBD
+}
+
 // ResolvedProvider normalizes an empty Provider to EBSProviderViperblockd,
 // the default provider.
 func (c EBSConfig) ResolvedProvider() string {
@@ -801,6 +807,27 @@ func validateVultrPool(cc *ClusterConfig, p ExternalPool) error {
 	return nil
 }
 
+// validateSingleEBSProvider refuses nodes whose explicit [nodes.<name>.ebs]
+// providers differ. A peer stanza without one is unknown, not the default: the
+// generated config omits it, and the cluster's KV lock is the authoritative check.
+func validateSingleEBSProvider(cc *ClusterConfig) error {
+	var first, want string
+	for _, name := range slices.Sorted(maps.Keys(cc.Nodes)) {
+		if !viper.IsSet("nodes." + name + ".ebs.provider") {
+			continue
+		}
+		got := cc.Nodes[name].EBS.ResolvedProvider()
+		if first == "" {
+			first, want = name, got
+			continue
+		}
+		if got != want {
+			return fmt.Errorf("config: [nodes.%s.ebs] provider resolves to %q but [nodes.%s.ebs] resolves to %q; every node in a cluster must use the same EBS provider", first, want, name, got)
+		}
+	}
+	return nil
+}
+
 func validateClusterConfig(cc *ClusterConfig) error {
 	if viper.IsSet("network.external_dhcp") {
 		return fmt.Errorf("config: [network] external_dhcp is no longer supported; remove the key (static WAN-pool allocation only)")
@@ -809,9 +836,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		if viper.IsSet("nodes." + nodeName + ".vpcd.dhcp_bind_bridge") {
 			return fmt.Errorf("config: [nodes.%s.vpcd] dhcp_bind_bridge is no longer supported; remove the key (vpcd no longer runs a DHCP client)", nodeName)
 		}
-		switch nodeCfg.EBS.Provider {
-		case "", EBSProviderViperblockd, EBSProviderQEMUNBD:
-		case EBSProviderEmbedded:
+		switch {
+		case nodeCfg.EBS.Provider == "", SupportedEBSProvider(nodeCfg.EBS.Provider):
+		case nodeCfg.EBS.Provider == EBSProviderEmbedded:
 			// Refuse rather than silently upgrade: the embedded engine is gone,
 			// and starting anyway would migrate this node's volumes to
 			// ebsmetadata one-way without the operator ever asking for it.
@@ -829,6 +856,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 			return fmt.Errorf("config: [nodes.%s.ebs] cache_size_mb=%d must not be negative; use 0 to disable the cache, or remove the key for the default of %d", nodeName, *c, DefaultCacheSizeMB)
 		}
 	}
+	if err := validateSingleEBSProvider(cc); err != nil {
+		return err
+	}
 
 	if err := validateNorthstarCluster(cc); err != nil {
 		return err

spinifex/bootstrap/config/config_test.go

@@ -1108,6 +1108,45 @@ provider = "bogus"
 	assert.Contains(t, err.Error(), `provider="bogus"`)
 }
 
+func TestLoadConfig_EBSProvider_SingleProviderPerCluster(t *testing.T) {
+	stanza := func(node, provider string) string {
+		return fmt.Sprintf("[nodes.%s.ebs]\nprovider = %q\n", node, provider)
+	}
+	cases := []struct {
+		name    string
+		ebs     []string
+		wantErr string
+	}{
+		{"all unset", []string{"", "", ""}, ""},
+		{"explicit viperblockd beside unset peers", []string{"", stanza("n2", EBSProviderViperblockd), ""}, ""},
+		{"explicit qemunbd beside unset peers", []string{stanza("n1", EBSProviderQEMUNBD), "", ""}, ""},
+		{"all explicit qemunbd", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderQEMUNBD), stanza("n3", EBSProviderQEMUNBD)}, ""},
+		{"two explicit providers differ", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderViperblockd), ""},
+			`[nodes.n1.ebs] provider resolves to "qemunbd" but [nodes.n2.ebs] resolves to "viperblockd"`},
+		{"only the third node differs", []string{stanza("n1", EBSProviderViperblockd), stanza("n2", EBSProviderViperblockd), stanza("n3", EBSProviderQEMUNBD)},
+			`[nodes.n1.ebs] provider resolves to "viperblockd" but [nodes.n3.ebs] resolves to "qemunbd"`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			body := "node = \"n1\"\n"
+			for i, ebs := range tc.ebs {
+				body += fmt.Sprintf("\n[nodes.n%d]\nregion = \"us-east-1\"\n%s", i+1, ebs)
+			}
+			resetViper(t)
+			path := filepath.Join(t.TempDir(), "spinifex.toml")
+			require.NoError(t, os.WriteFile(path, []byte(body), 0600))
+
+			_, err := LoadConfig(path)
+			if tc.wantErr == "" {
+				require.NoError(t, err)
+				return
+			}
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.wantErr)
+		})
+	}
+}
+
 func TestEBSConfig_ResolvedTunablesDefault(t *testing.T) {
 	var c EBSConfig
 	assert.Equal(t, DefaultNBDKitThreads, c.ResolvedThreads())

spinifex/domains/ec2/accountcleanup/cleanup.go

@@ -69,6 +69,28 @@ var (
 	_ TagIndexOwner       = (*ec2tags.TagsServiceImpl)(nil)
 )
 
+// kindPrefixes maps each cleanable kind to its resource-ID prefix in the tag index.
+var kindPrefixes = map[string]string{
+	ec2v1.PlatformKindKeyPair:        keyPairIDPrefix,
+	ec2v1.PlatformKindPlacementGroup: placementGroupIDPrefix,
+	ec2v1.PlatformKindLaunchTemplate: launchTemplateIDPrefix,
+}
+
+// selectKinds returns the kinds a request cleans, in report order. No kinds
+// means all three; a kind that is not cleanable, such as the tag index, is refused.
+func selectKinds(requested []string) ([]string, error) {
+	all := []string{ec2v1.PlatformKindKeyPair, ec2v1.PlatformKindPlacementGroup, ec2v1.PlatformKindLaunchTemplate}
+	if len(requested) == 0 {
+		return all, nil
+	}
+	for _, kind := range requested {
+		if !slices.Contains(all, kind) {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The kind is not one of key-pair, placement-group or launch-template.")
+		}
+	}
+	return slices.DeleteFunc(all, func(kind string) bool { return !slices.Contains(requested, kind) }), nil
+}
+
 // DefaultSettleWindow is how old headerless residue must be before it is
 // removed; anything younger may be a create still in flight.
 const DefaultSettleWindow = 10 * time.Minute
@@ -102,6 +124,7 @@ const (
 // CleanupPlatform deletes the account's key pairs, empty placement groups and
 // launch templates, removes settled residue and orphaned tag-index entries,
 // then re-scans every owner store to build the result. It is safe to re-run.
+// Request.Kinds limits the run to those kinds and their tag-index entries.
 func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, error) {
 	if req == nil || req.AccountID == "" {
 		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "The request must specify an account ID.")
@@ -118,27 +141,54 @@ func (c *Cleanup) CleanupPlatform(ctx context.Context, req *Request) (*Result, e
 	acct := req.AccountID
 	res := &Result{AccountID: acct, CorrelationRef: req.CorrelationRef}
 
-	refs := c.readReferences(ctx, acct)
-	keyRun := c.cleanKeyPairs(ctx, acct, settledBefore, refs)
-	groupRun := c.cleanPlacementGroups(ctx, acct, refs)
-	templateRun := c.cleanLaunchTemplates(ctx, acct, settledBefore)
+	kinds, err := selectKinds(req.Kinds)
+	if err != nil {
+		return nil, err
+	}
+	var prefixes []string
+	for _, kind := range kinds {
+		prefixes = append(prefixes, kindPrefixes[kind])
+	}
+
+	var keyRun *keyPairRun
+	var groupRun *placementGroupRun
+	var templateRun *launchTemplateRun
+	if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) || slices.Contains(kinds, ec2v1.PlatformKindPlacementGroup) {
+		refs := c.readReferences(ctx, acct)
+		if slices.Contains(kinds, ec2v1.PlatformKindKeyPair) {
+			keyRun = c.cleanKeyPairs(ctx, acct, settledBefore, refs)
+		}
+		if slices.Contains(kinds, ec2v1.PlatformKindPlacementGroup) {
+			groupRun = c.cleanPlacementGroups(ctx, acct, refs)
+		}
+	}
+	if slices.Contains(kinds, ec2v1.PlatformKindLaunchTemplate) {
+		templateRun = c.cleanLaunchTemplates(ctx, acct, settledBefore)
+	}
 
 	// Tags are listed before the owner re-scans, so a resource created in
 	// between is seen as owned and keeps its tags.
 	tagKind := c.startKind(ec2v1.PlatformKindTagIndex)
-	tagged, tagErr := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	tagged, tagErr := c.tags.ScanAccountResourceIDs(ctx, acct, prefixes...)
 	if tagErr != nil {
 		slog.WarnContext(ctx, "Platform cleanup: tag index listing failed", "err", tagErr)
 		tagKind.incomplete(ec2v1.PlatformReasonStoreListFailed)
 	}
 
-	owners := map[string]ownedIDs{
-		keyPairIDPrefix:        c.finishKeyPairs(ctx, acct, keyRun, res),
-		placementGroupIDPrefix: c.finishPlacementGroups(ctx, acct, groupRun, res),
-		launchTemplateIDPrefix: c.finishLaunchTemplates(ctx, acct, templateRun, res),
+	owners := map[string]ownedIDs{}
+	if keyRun != nil {
+		owners[keyPairIDPrefix] = c.finishKeyPairs(ctx, acct, keyRun, res)
+		res.Kinds = append(res.Kinds, keyRun.kind.report)
 	}
-	res.Kinds = []KindReport{keyRun.kind.report, groupRun.kind.report, templateRun.kind.report}
-	res.Kinds = append(res.Kinds, c.cleanTagIndex(ctx, acct, tagKind, tagged, tagErr == nil, owners, res))
+	if groupRun != nil {
+		owners[placementGroupIDPrefix] = c.finishPlacementGroups(ctx, acct, groupRun, res)
+		res.Kinds = append(res.Kinds, groupRun.kind.report)
+	}
+	if templateRun != nil {
+		owners[launchTemplateIDPrefix] = c.finishLaunchTemplates(ctx, acct, templateRun, res)
+		res.Kinds = append(res.Kinds, templateRun.kind.report)
+	}
+	res.Kinds = append(res.Kinds, c.cleanTagIndex(ctx, acct, tagKind, tagged, prefixes, tagErr == nil, owners, res))
 
 	slog.InfoContext(ctx, "Platform cleanup completed", "accountID", acct, "correlationRef", req.CorrelationRef,
 		"items", len(res.Items), "duration_ms", otelsetup.Millis(time.Since(start)))

spinifex/domains/ec2/accountcleanup/cleanup_test.go

@@ -775,3 +775,38 @@ func (f failingResidue) CleanupAccountResidue(ctx context.Context, acct string,
 	}
 	return &ec2launchtemplate.AccountCleanup{KeptTemplateIDs: scan.LiveTemplateIDs, OrphanBodiesFailed: scan.OrphanVersionBodies}, nil
 }
+
+// A run limited to one kind cleans that kind and its tag-index entries only:
+// the other kinds keep their resources and their tags, and are not reported.
+func TestCleanupPlatformLimitedToKindsLeavesTheRestAlone(t *testing.T) {
+	o := newOwners(t)
+	keyID := o.importKey(t, testAccount, "k1")
+	groupID := o.createGroup(t, testAccount, "g1")
+	o.tag(t, testAccount, keyID)
+
+	c := o.newCleanup()
+	c.now = func() time.Time { return time.Now().Add(time.Hour) }
+	res, err := c.CleanupPlatform(t.Context(), &Request{AccountID: testAccount, Kinds: []string{ec2v1.PlatformKindPlacementGroup}})
+	require.NoError(t, err)
+
+	assert.Equal(t, []Item{item(groupID, ec2v1.PlatformKindPlacementGroup, DispositionDeleted, "")}, res.Items)
+	assert.Equal(t, map[string]string{
+		ec2v1.PlatformKindPlacementGroup: ec2v1.PlatformStatusObservedClean,
+		ec2v1.PlatformKindTagIndex:       ec2v1.PlatformStatusObservedClean,
+	}, statuses(res))
+
+	keys, err := o.keys.ScanAccount(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, keys.LiveKeyPairIDs, "an unselected kind is not touched")
+	tagged, err := o.tags.ScanAccountResourceIDs(t.Context(), testAccount, "key-")
+	require.NoError(t, err)
+	assert.Equal(t, []string{keyID}, tagged, "an unselected kind keeps its tags")
+}
+
+func TestCleanupPlatformRefusesAKindItCannotClean(t *testing.T) {
+	o := newOwners(t)
+	for _, kinds := range [][]string{{ec2v1.PlatformKindTagIndex}, {"volume"}} {
+		_, err := o.newCleanup().CleanupPlatform(t.Context(), &Request{AccountID: testAccount, Kinds: kinds})
+		require.Error(t, err, "%v", kinds)
+	}
+}

spinifex/domains/ec2/accountcleanup/kinds.go

@@ -282,15 +282,15 @@ func (c *Cleanup) finishLaunchTemplates(ctx context.Context, acct string, run *l
 // cleanTagIndex clears entries whose owner re-scan resolved and holds no
 // record for the ID, then re-lists the index to prove each clear. Entries of
 // resources that still exist are theirs, not residue, and are left alone.
-func (c *Cleanup) cleanTagIndex(ctx context.Context, acct string, kind *kindRun, tagged []string, listed bool, owners map[string]ownedIDs, res *Result) KindReport {
+func (c *Cleanup) cleanTagIndex(ctx context.Context, acct string, kind *kindRun, tagged, prefixes []string, listed bool, owners map[string]ownedIDs, res *Result) KindReport {
 	if listed {
-		c.clearOrphanTags(ctx, acct, kind, tagged, owners, res)
+		c.clearOrphanTags(ctx, acct, kind, tagged, prefixes, owners, res)
 	}
 	c.finishKind(kind)
 	return kind.report
 }
 
-func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRun, tagged []string, owners map[string]ownedIDs, res *Result) {
+func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRun, tagged, prefixes []string, owners map[string]ownedIDs, res *Result) {
 	// A resource this run reports as not deleted owns its tags even when its
 	// kind's re-scan failed.
 	reported := map[string]bool{}
@@ -317,7 +317,7 @@ func (c *Cleanup) clearOrphanTags(ctx context.Context, acct string, kind *kindRu
 			cleared = append(cleared, id)
 		}
 	}
-	after, err := c.tags.ScanAccountResourceIDs(ctx, acct, keyPairIDPrefix, placementGroupIDPrefix, launchTemplateIDPrefix)
+	after, err := c.tags.ScanAccountResourceIDs(ctx, acct, prefixes...)
 	if err != nil {
 		slog.WarnContext(ctx, "Platform cleanup: tag index re-scan failed", "err", err)
 		kind.incomplete(ec2v1.PlatformReasonStoreListFailed)

spinifex/domains/ec2/accountcleanup/types.go

@@ -7,6 +7,9 @@ import "time"
 type Request struct {
 	AccountID      string `json:"account_id"`
 	CorrelationRef string `json:"correlation_ref,omitempty"`
+
+	// Kinds limits the run to these platform kinds. Empty means all three.
+	Kinds []string `json:"kinds,omitempty"`
 }
 
 // Result is what the owner stores hold after one run. Kinds reuse the ec2.v1

spinifex/domains/ec2/instance/service_nats.go

@@ -1,80 +0,0 @@
-package instance
-
-import (
-	"context"
-	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/nats-io/nats.go"
-)
-
-// NATSInstanceService handles instance operations via NATS. Fan-out methods
-// (DescribeInstances, DescribeInstanceTypes) intentionally error — use the
-// gateway's scatter-gather helper for cluster-wide results.
-type NATSInstanceService struct {
-	natsConn *nats.Conn
-}
-
-var _ InstanceService = (*NATSInstanceService)(nil)
-
-// NewNATSInstanceService creates a new NATS-based instance service.
-func NewNATSInstanceService(conn *nats.Conn) InstanceService {
-	return &NATSInstanceService{natsConn: conn}
-}
-
-func (s *NATSInstanceService) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
-	if input == nil || input.InstanceType == nil {
-		return nil, fmt.Errorf("instance type is required")
-	}
-	topic := fmt.Sprintf("ec2.RunInstances.%s", aws.StringValue(input.InstanceType))
-	return natsmsg.NATSRequest[ec2.Reservation](ctx, s.natsConn, topic, input, 5*time.Minute, accountID)
-}
-
-// DescribeInstances is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstances(_ context.Context, _ *ec2.DescribeInstancesInput, _ string) (*ec2.DescribeInstancesOutput, error) {
-	return nil, fmt.Errorf("DescribeInstances is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-// DescribeInstanceTypes is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstanceTypes(_ context.Context, _ *ec2.DescribeInstanceTypesInput, _ string) (*DescribeInstanceTypesNodeOutput, error) {
-	return nil, fmt.Errorf("DescribeInstanceTypes is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-// DescribeInstanceStatus is a fan-out operation; use the gateway's scatter-gather
-// helper instead of this single-node client.
-func (s *NATSInstanceService) DescribeInstanceStatus(_ context.Context, _ *ec2.DescribeInstanceStatusInput, _ string) (*ec2.DescribeInstanceStatusOutput, error) {
-	return nil, fmt.Errorf("DescribeInstanceStatus is fan-out; use gateway scatter-gather, not NATSInstanceService")
-}
-
-func (s *NATSInstanceService) DescribeInstanceAttribute(ctx context.Context, input *ec2.DescribeInstanceAttributeInput, accountID string) (*ec2.DescribeInstanceAttributeOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstanceAttributeOutput](ctx, s.natsConn, "ec2.DescribeInstanceAttribute", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) DescribeStoppedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstancesOutput](ctx, s.natsConn, "ec2.DescribeStoppedInstances", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) DescribeTerminatedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error) {
-	return natsmsg.NATSRequest[ec2.DescribeInstancesOutput](ctx, s.natsConn, "ec2.DescribeTerminatedInstances", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) ModifyInstanceAttribute(ctx context.Context, input *ec2.ModifyInstanceAttributeInput, accountID string) (*ec2.ModifyInstanceAttributeOutput, error) {
-	return natsmsg.NATSRequest[ec2.ModifyInstanceAttributeOutput](ctx, s.natsConn, "ec2.ModifyInstanceAttribute", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) ModifyInstanceMetadataOptions(ctx context.Context, input *ec2.ModifyInstanceMetadataOptionsInput, accountID string) (*ec2.ModifyInstanceMetadataOptionsOutput, error) {
-	return natsmsg.NATSRequest[ec2.ModifyInstanceMetadataOptionsOutput](ctx, s.natsConn, "ec2.ModifyInstanceMetadataOptions", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) StartStoppedInstance(ctx context.Context, input *StartStoppedInstanceInput, accountID string) (*StartStoppedInstanceOutput, error) {
-	return natsmsg.NATSRequest[StartStoppedInstanceOutput](ctx, s.natsConn, "ec2.StartStoppedInstance", input, 30*time.Second, accountID)
-}
-
-func (s *NATSInstanceService) TerminateStoppedInstance(ctx context.Context, input *TerminateStoppedInstanceInput, accountID string) (*TerminateStoppedInstanceOutput, error) {
-	return natsmsg.NATSRequest[TerminateStoppedInstanceOutput](ctx, s.natsConn, "ec2.TerminateStoppedInstance", input, 30*time.Second, accountID)
-}

spinifex/domains/ec2/instance/service_nats_test.go

@@ -1,364 +0,0 @@
-package instance
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/nats-io/nats-server/v2/server"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// startTestNATSServer starts an embedded NATS server for testing.
-func startTestNATSServer(t *testing.T) (*server.Server, string) {
-	t.Helper()
-	ns, nc := testutil.StartTestNATS(t)
-	_ = nc // connection managed by testutil cleanup
-	return ns, ns.ClientURL()
-}
-
-// createValidRunInstancesInput creates a valid RunInstancesInput for testing.
-func createValidRunInstancesInput() *ec2.RunInstancesInput {
-	return &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String("t3.micro"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-		KeyName:      aws.String("test-key-pair"),
-		SecurityGroupIds: []*string{
-			aws.String("sg-0123456789abcdef0"),
-		},
-		SubnetId: aws.String("subnet-6e7f829e"),
-		UserData: aws.String("#!/bin/bash\necho 'test'"),
-	}
-}
-
-// createValidReservation creates a valid ec2.Reservation response for testing.
-func createValidReservation() *ec2.Reservation {
-	return &ec2.Reservation{
-		ReservationId: aws.String("r-0123456789abcdef0"),
-		OwnerId:       aws.String("123456789012"),
-		Instances: []*ec2.Instance{
-			{
-				InstanceId:   aws.String("i-0123456789abcdef0"),
-				InstanceType: aws.String("t3.micro"),
-				ImageId:      aws.String("ami-0abcdef1234567890"),
-				State: &ec2.InstanceState{
-					Code: aws.Int64(0),
-					Name: aws.String("pending"),
-				},
-				PrivateIpAddress: aws.String("10.0.1.100"),
-				SubnetId:         aws.String("subnet-6e7f829e"),
-			},
-		},
-	}
-}
-
-// TestNATSInstanceService_RunInstances_Success tests successful RunInstances operation.
-func TestNATSInstanceService_RunInstances_Success(t *testing.T) {
-	// Skip if LOG_IGNORE is set
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	// Start test NATS server
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	// Create NATS connections
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err, "Failed to connect to NATS")
-	defer nc.Close()
-
-	// Create mock daemon subscriber that responds with valid reservation
-	// Subscribe to the per-instance-type topic (t3.micro from createValidRunInstancesInput)
-	mockReservation := createValidReservation()
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		// Validate that request is properly formatted
-		var input ec2.RunInstancesInput
-		err := json.Unmarshal(msg.Data, &input)
-		if err != nil {
-			t.Errorf("Mock daemon received invalid JSON: %v", err)
-			return
-		}
-
-		// Send back valid reservation
-		responseData, _ := json.Marshal(mockReservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err, "Failed to subscribe mock daemon")
-
-	// Create NATSInstanceService
-	service := NewNATSInstanceService(nc)
-	require.NotNil(t, service)
-
-	// Create valid input
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify success
-	require.NoError(t, err, "RunInstances should succeed")
-	require.NotNil(t, reservation, "Reservation should not be nil")
-
-	// Verify reservation contents
-	assert.Equal(t, *mockReservation.ReservationId, *reservation.ReservationId)
-	assert.Equal(t, *mockReservation.OwnerId, *reservation.OwnerId)
-	require.Len(t, reservation.Instances, 1)
-
-	instance := reservation.Instances[0]
-	assert.Equal(t, "i-0123456789abcdef0", *instance.InstanceId)
-	assert.Equal(t, "t3.micro", *instance.InstanceType)
-	assert.Equal(t, int64(0), *instance.State.Code)
-	assert.Equal(t, "pending", *instance.State.Name)
-}
-
-// TestNATSInstanceService_RunInstances_DaemonError tests error handling when daemon returns error.
-func TestNATSInstanceService_RunInstances_DaemonError(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that responds with error
-	// Subscribe to the per-instance-type topic for the requested type
-	_, err = nc.QueueSubscribe("ec2.RunInstances.invalid.type", "spinifex-workers", func(msg *nats.Msg) {
-		// Send back error response
-		errorResponse := awserrors.GenerateErrorPayload("InvalidInstanceType")
-		msg.Respond(errorResponse)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-
-	input := &ec2.RunInstancesInput{
-		ImageId:      aws.String("ami-0abcdef1234567890"),
-		InstanceType: aws.String("invalid.type"),
-		MinCount:     aws.Int64(1),
-		MaxCount:     aws.Int64(1),
-	}
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify error handling - error should be just the AWS error code for gateway lookup
-	require.Error(t, err, "Should return error when daemon returns error")
-	assert.Nil(t, reservation, "Reservation should be nil on error")
-	assert.Equal(t, "InvalidInstanceType", err.Error(), "Error should be the AWS error code")
-}
-
-// TestNATSInstanceService_RunInstances_NoSubscriber tests behavior when no daemon is subscribed.
-func TestNATSInstanceService_RunInstances_NoSubscriber(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// No subscriber - no daemon running
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	start := time.Now()
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-	duration := time.Since(start)
-
-	// Verify error behavior
-	// When no subscribers are available, NATS returns ErrNoResponders immediately
-	require.Error(t, err, "Should fail when no subscriber")
-	assert.Nil(t, reservation, "Reservation should be nil")
-	assert.ErrorIs(t, err, nats.ErrNoResponders, "Error should be ErrNoResponders")
-
-	// Should fail quickly (within 1 second), not wait for 30 second timeout
-	assert.Less(t, duration.Seconds(), 1.0, "Should fail immediately when no responders")
-}
-
-// TestNATSInstanceService_RunInstances_InvalidResponse tests handling of malformed responses.
-func TestNATSInstanceService_RunInstances_InvalidResponse(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that responds with invalid JSON
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		// Send back malformed JSON
-		msg.Respond([]byte(`{"invalid": json response`))
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Call RunInstances
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify error handling
-	require.Error(t, err, "Should return error for invalid JSON")
-	assert.Nil(t, reservation, "Reservation should be nil")
-	assert.Contains(t, err.Error(), "failed to unmarshal", "Error should indicate unmarshal failure")
-}
-
-// TestNATSInstanceService_RunInstances_MarshalError tests handling when input marshaling fails.
-func TestNATSInstanceService_RunInstances_MarshalError(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	service := NewNATSInstanceService(nc)
-
-	// Nil input should be rejected before hitting NATS
-	reservation, err := service.RunInstances(context.Background(), nil, "123456789012")
-	require.Error(t, err, "Should handle nil input")
-	assert.Nil(t, reservation)
-	assert.Contains(t, err.Error(), "instance type is required")
-}
-
-// TestNATSInstanceService_RunInstances_MultipleInstances tests launching multiple instances.
-func TestNATSInstanceService_RunInstances_MultipleInstances(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create mock daemon that returns multiple instances
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		var input ec2.RunInstancesInput
-		json.Unmarshal(msg.Data, &input)
-
-		// Create reservation with count matching MinCount
-		count := int(*input.MinCount)
-		instances := make([]*ec2.Instance, count)
-		for i := range count {
-			instanceId := aws.String(aws.StringValue(aws.String(fmt.Sprintf("i-%d", i))))
-			instances[i] = &ec2.Instance{
-				InstanceId:   instanceId,
-				InstanceType: input.InstanceType,
-				ImageId:      input.ImageId,
-				State: &ec2.InstanceState{
-					Code: aws.Int64(0),
-					Name: aws.String("pending"),
-				},
-			}
-		}
-
-		reservation := &ec2.Reservation{
-			ReservationId: aws.String("r-0123456789abcdef0"),
-			Instances:     instances,
-		}
-
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-
-	// Request 3 instances
-	input := createValidRunInstancesInput()
-	input.MinCount = aws.Int64(3)
-	input.MaxCount = aws.Int64(3)
-
-	reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-
-	// Verify multiple instances returned
-	require.NoError(t, err)
-	require.NotNil(t, reservation)
-	assert.Len(t, reservation.Instances, 3, "Should have 3 instances")
-}
-
-// TestNATSInstanceService_RunInstances_QueueGroup tests load balancing with queue groups.
-func TestNATSInstanceService_RunInstances_QueueGroup(t *testing.T) {
-	if os.Getenv("LOG_IGNORE") != "" {
-		t.Setenv("LOG_IGNORE", "1")
-	}
-
-	ns, natsURL := startTestNATSServer(t)
-	defer ns.Shutdown()
-
-	nc, err := nats.Connect(natsURL)
-	require.NoError(t, err)
-	defer nc.Close()
-
-	// Create multiple mock daemon workers in same queue group
-	worker1Count := 0
-	worker2Count := 0
-
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		worker1Count++
-		reservation := createValidReservation()
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	_, err = nc.QueueSubscribe("ec2.RunInstances.t3.micro", "spinifex-workers", func(msg *nats.Msg) {
-		worker2Count++
-		reservation := createValidReservation()
-		responseData, _ := json.Marshal(reservation)
-		msg.Respond(responseData)
-	})
-	require.NoError(t, err)
-
-	service := NewNATSInstanceService(nc)
-	input := createValidRunInstancesInput()
-
-	// Send multiple requests
-	requestCount := 10
-	for range requestCount {
-		reservation, err := service.RunInstances(context.Background(), input, "123456789012")
-		require.NoError(t, err)
-		require.NotNil(t, reservation)
-	}
-
-	// Verify load was distributed (both workers should have handled some requests)
-	// With queue groups, NATS distributes messages across subscribers
-	t.Logf("Worker 1 handled: %d requests", worker1Count)
-	t.Logf("Worker 2 handled: %d requests", worker2Count)
-
-	assert.Positive(t, worker1Count, "Worker 1 should handle some requests")
-	assert.Positive(t, worker2Count, "Worker 2 should handle some requests")
-	assert.Equal(t, requestCount, worker1Count+worker2Count, "All requests should be handled")
-}

spinifex/domains/lambda/awsapi/get.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"log/slog"
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -18,47 +19,109 @@ import (
 // codeURLTTL is how long the presigned code location stays valid.
 const codeURLTTL = 10 * time.Minute
 
+// maxListItems is the largest MaxItems a list action accepts.
+const maxListItems = 10000
+
 func notFound(region, accountID, name string) error {
 	return awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "Function not found: %s",
 		arn.FormatLambdaFunction(region, accountID, name))
 }
 
-// getFunction implements GetFunction for the $LATEST version.
-func (d Deps) getFunction(ctx context.Context, c *call) (reply, error) {
+// lookup resolves the requested $LATEST function's record; query is the
+// Qualifier query parameter, empty for an action that has none.
+func (d Deps) lookup(ctx context.Context, c *call, query string) (function.Record, uint64, error) {
 	name, q, err := resolveFunctionName(c.Region, c.AccountID, c.Params[0])
 	if err != nil {
-		return reply{}, err
+		return function.Record{}, 0, err
 	}
-	if query := c.Query.Get("Qualifier"); query != "" {
+	if query != "" {
 		if q != "" && q != query {
-			return reply{}, invalid("The derived qualifier from the function name does not match the specified qualifier.")
+			return function.Record{}, 0, invalid("The derived qualifier from the function name does not match the specified qualifier.")
 		}
 		q = query
 	}
 	if q != "" && q != "$LATEST" {
-		return reply{}, notFound(c.Region, c.AccountID, name+":"+q)
+		return function.Record{}, 0, notFound(c.Region, c.AccountID, name+":"+q)
 	}
 	rec, revision, err := d.Meta.Get(ctx, c.AccountID, c.Region, name)
 	if errors.Is(err, function.ErrNotFound) {
-		return reply{}, notFound(c.Region, c.AccountID, name)
+		return function.Record{}, 0, notFound(c.Region, c.AccountID, name)
 	}
+	return rec, revision, err
+}
+
+// loadConfiguration resolves the requested $LATEST function and renders its
+// configuration; it is a metadata read and never touches the code store.
+func (d Deps) loadConfiguration(ctx context.Context, c *call, query string) (function.Record, *lambda.FunctionConfiguration, error) {
+	rec, revision, err := d.lookup(ctx, c, query)
 	if err != nil {
-		return reply{}, err
+		return function.Record{}, nil, err
 	}
 
 	var env map[string]string
 	if rec.EnvCipher != "" {
 		if d.AEAD == nil {
-			return reply{}, serviceError("environment decryption is unavailable")
+			return function.Record{}, nil, serviceError("environment decryption is unavailable")
 		}
 		if env, err = function.OpenEnv(d.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher); err != nil {
 			slog.ErrorContext(ctx, "Lambda: open environment failed", "function_id", rec.FunctionID, "err", err)
-			return reply{}, serviceError("could not read the function environment")
+			return function.Record{}, nil, serviceError("could not read the function environment")
 		}
 	}
 	cfg, err := configuration(rec, revision, env)
 	if err != nil {
-		return reply{}, serviceError("could not build the function configuration")
+		return function.Record{}, nil, serviceError("could not build the function configuration")
+	}
+	return rec, cfg, nil
+}
+
+// getFunctionConfiguration implements GetFunctionConfiguration for $LATEST.
+func (d Deps) getFunctionConfiguration(ctx context.Context, c *call) (reply, error) {
+	_, cfg, err := d.loadConfiguration(ctx, c, c.Query.Get("Qualifier"))
+	if err != nil {
+		return reply{}, err
+	}
+	return reply{Status: http.StatusOK, Body: cfg}, nil
+}
+
+// noCodeSigning is AWS's body for a function without a code signing config:
+// both members present and null, which the SDK shapes cannot encode.
+var noCodeSigning = rawJSON(`{"CodeSigningConfigArn":null,"FunctionName":null}`)
+
+// getFunctionCodeSigningConfig implements GetFunctionCodeSigningConfig; code
+// signing is unsupported, so it only proves the function exists.
+func (d Deps) getFunctionCodeSigningConfig(ctx context.Context, c *call) (reply, error) {
+	if _, _, err := d.lookup(ctx, c, ""); err != nil {
+		return reply{}, err
+	}
+	return reply{Status: http.StatusOK, Body: noCodeSigning}, nil
+}
+
+// listVersionsByFunction implements ListVersionsByFunction while $LATEST is
+// the only version. Paging is refused rather than guessed at.
+func (d Deps) listVersionsByFunction(ctx context.Context, c *call) (reply, error) {
+	if raw, ok := c.Query["MaxItems"]; ok {
+		n, err := strconv.Atoi(raw[0])
+		if err != nil || n < 1 || n > maxListItems {
+			return reply{}, invalid("MaxItems must be between 1 and %d", maxListItems)
+		}
+	}
+	if c.Query.Has("Marker") {
+		return reply{}, unsupported("Marker")
+	}
+	rec, cfg, err := d.loadConfiguration(ctx, c, "")
+	if err != nil {
+		return reply{}, err
+	}
+	cfg.FunctionArn = aws.String(arn.FormatLambdaFunction(rec.Region, rec.AccountID, rec.FunctionName) + ":$LATEST")
+	return reply{Status: http.StatusOK, Body: &lambda.ListVersionsByFunctionOutput{Versions: []*lambda.FunctionConfiguration{cfg}}}, nil
+}
+
+// getFunction implements GetFunction for the $LATEST version.
+func (d Deps) getFunction(ctx context.Context, c *call) (reply, error) {
+	rec, cfg, err := d.loadConfiguration(ctx, c, c.Query.Get("Qualifier"))
+	if err != nil {
+		return reply{}, err
 	}
 	out := &lambda.GetFunctionOutput{
 		Configuration: cfg,

spinifex/domains/lambda/awsapi/handler_test.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"runtime"
 	"strings"
 	"sync"
@@ -662,8 +663,316 @@ func TestAuthorization(t *testing.T) {
 
 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateFunction", "GetFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "ListVersionsByFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
+	assert.Equal(t, "GetFunctionCodeSigningConfig", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2020-06-30/functions/x/code-signing-config", nil)))
 	assert.Equal(t, "CreateFunction", reg.ResolveAction(httptest.NewRequest(http.MethodPost, "/2015-03-31/functions", nil)))
 	assert.Equal(t, "GetFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x", nil)))
+	assert.Equal(t, "GetFunctionConfiguration", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/configuration", nil)))
 	assert.Empty(t, reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x", nil)))
 }
+
+func TestGetFunctionConfiguration_ReturnsTheSameConfigurationAsGetFunction(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+
+	rec, err := e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+	require.NoError(t, err)
+	var full lambda.GetFunctionOutput
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &full))
+
+	rec, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, rec.Code)
+	assert.NotContains(t, rec.Body.String(), `"Code"`, "the configuration is the top-level body, not GetFunction's wrapper")
+	var cfg lambda.FunctionConfiguration
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &cfg))
+	assert.Equal(t, *full.Configuration, cfg)
+	assert.Equal(t, "prod", *cfg.Environment.Variables["MODE"])
+
+	arn := "arn%3Aaws%3Alambda%3A" + gwtest.Region + "%3A" + gwtest.AccountID + "%3Afunction%3Ahello"
+	for _, path := range []string{
+		"/2015-03-31/functions/hello/configuration?Qualifier=%24LATEST",
+		"/2015-03-31/functions/" + arn + "/configuration",
+		"/2015-03-31/functions/" + arn + "%3A%24LATEST/configuration",
+	} {
+		rec, err := e.do(http.MethodGet, path, nil)
+		require.NoError(t, err, path)
+		assert.Equal(t, http.StatusOK, rec.Code, path)
+	}
+}
+
+func TestGetFunctionConfiguration_MissingAndUnsupportedMatchGetFunction(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	for label, tc := range map[string]struct{ suffix, code string }{
+		"missing":         {"ghost", awserrors.ErrorResourceNotFoundException},
+		"version":         {"hello?Qualifier=3", awserrors.ErrorResourceNotFoundException},
+		"other account":   {"arn%3Aaws%3Alambda%3A" + gwtest.Region + "%3A999999999999%3Afunction%3Ahello", awserrors.ErrorResourceNotFoundException},
+		"qualifier clash": {"hello%3A%24LATEST?Qualifier=3", awserrors.ErrorInvalidParameterValueException},
+		"not lambda":      {"arn%3Aaws%3As3%3A%3A%3Abucket", awserrors.ErrorInvalidParameterValueException},
+	} {
+		t.Run(label, func(t *testing.T) {
+			path, query, _ := strings.Cut(tc.suffix, "?")
+			if query != "" {
+				query = "?" + query
+			}
+			_, getErr := e.do(http.MethodGet, "/2015-03-31/functions/"+path+query, nil)
+			requireCode(t, getErr, tc.code)
+			_, err := e.do(http.MethodGet, "/2015-03-31/functions/"+path+"/configuration"+query, nil)
+			requireCode(t, err, tc.code)
+			assert.Equal(t, getErr.Error(), err.Error())
+		})
+	}
+
+	e.h.Region = "us-west-2"
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestGetFunctionConfiguration_DoesNotNeedTheCodeStore(t *testing.T) {
+	for name, codes := range map[string]awsapi.CodeStoreFunc{
+		"store outage": func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") },
+		"no store":     nil,
+	} {
+		t.Run(name, func(t *testing.T) {
+			e := newEnv(t)
+			_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+			require.NoError(t, err)
+			e.deps.Codes = codes
+
+			rec, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+			require.NoError(t, err)
+			require.Equal(t, http.StatusOK, rec.Code)
+			var cfg lambda.FunctionConfiguration
+			require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &cfg))
+			assert.Equal(t, "hello", *cfg.FunctionName)
+
+			_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+			requireCode(t, err, awserrors.ErrorServiceException)
+		})
+	}
+}
+
+func TestGetFunctionConfiguration_Authorization(t *testing.T) {
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	setup := func(t *testing.T, statements ...iamdomain.Statement) *env {
+		e := newEnv(t, statements...)
+		_, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+			FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+			Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+		})
+		require.NoError(t, err)
+		return e
+	}
+	get := func(e *env) error {
+		_, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+		return err
+	}
+
+	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))))
+	})
+	t.Run("a deny on the function ARN applies", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Deny", "lambda:GetFunctionConfiguration", arn))))
+	})
+	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionConfiguration", arn+"-other"))))
+	})
+	t.Run("an allow on the function ARN passes", func(t *testing.T) {
+		require.NoError(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionConfiguration", arn))))
+	})
+}
+
+func TestGetFunctionCodeSigningConfig_ExistingFunctionHasExplicitNulls(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	arn := "arn%3Aaws%3Alambda%3A" + gwtest.Region + "%3A" + gwtest.AccountID + "%3Afunction%3Ahello"
+	for _, name := range []string{"hello", arn, gwtest.AccountID + "%3Afunction%3Ahello"} {
+		rec, err := e.do(http.MethodGet, "/2020-06-30/functions/"+name+"/code-signing-config", nil)
+		require.NoError(t, err, name)
+		require.Equal(t, http.StatusOK, rec.Code, name)
+		assert.Equal(t, "application/json", rec.Header().Get("Content-Type"))
+		assert.JSONEq(t, `{"CodeSigningConfigArn":null,"FunctionName":null}`, rec.Body.String(), name)
+	}
+}
+
+func TestGetFunctionCodeSigningConfig_MissingFunctionIsNotFound(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	_, err = e.do(http.MethodGet, "/2020-06-30/functions/ghost/code-signing-config", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, getErr := e.do(http.MethodGet, "/2015-03-31/functions/ghost", nil)
+	assert.Equal(t, getErr.Error(), err.Error())
+
+	e.h.Region = "us-west-2"
+	_, err = e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestGetFunctionCodeSigningConfig_NeedsNeitherCodeStoreNorDecryption(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+	e.deps.Codes = func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") }
+	e.deps.AEAD = nil
+
+	rec, err := e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{"CodeSigningConfigArn":null,"FunctionName":null}`, rec.Body.String())
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	requireCode(t, err, awserrors.ErrorServiceException)
+
+	e.deps.Codes = nil
+	_, err = e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+	require.NoError(t, err)
+}
+
+func TestGetFunctionCodeSigningConfig_Authorization(t *testing.T) {
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	setup := func(t *testing.T, statements ...iamdomain.Statement) *env {
+		e := newEnv(t, statements...)
+		_, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+			FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+			Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+		})
+		require.NoError(t, err)
+		return e
+	}
+	get := func(e *env) error {
+		_, err := e.do(http.MethodGet, "/2020-06-30/functions/hello/code-signing-config", nil)
+		return err
+	}
+
+	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))))
+	})
+	t.Run("a deny on the function ARN applies", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Deny", "lambda:GetFunctionCodeSigningConfig", arn))))
+	})
+	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
+		gwtest.AssertDenied(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionCodeSigningConfig", arn+"-other"))))
+	})
+	t.Run("an allow on the function ARN passes", func(t *testing.T) {
+		require.NoError(t, get(setup(t, gwtest.Statement("Allow", "lambda:GetFunctionCodeSigningConfig", arn))))
+	})
+}
+
+func TestListVersionsByFunction_ListsOnlyLatest(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+	rec, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/configuration", nil)
+	require.NoError(t, err)
+	var cfg lambda.FunctionConfiguration
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &cfg))
+
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	for _, path := range []string{
+		"/2015-03-31/functions/hello/versions",
+		"/2015-03-31/functions/hello/versions?MaxItems=10000",
+		"/2015-03-31/functions/hello/versions?MaxItems=1",
+		"/2015-03-31/functions/" + url.PathEscape(arn) + "/versions",
+	} {
+		rec, err := e.do(http.MethodGet, path, nil)
+		require.NoError(t, err, path)
+		require.Equal(t, http.StatusOK, rec.Code, path)
+		assert.NotContains(t, rec.Body.String(), "NextMarker", path)
+		var out lambda.ListVersionsByFunctionOutput
+		require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out), path)
+		require.Len(t, out.Versions, 1, path)
+		v := out.Versions[0]
+		assert.Equal(t, arn+":$LATEST", *v.FunctionArn, path)
+		assert.Equal(t, "$LATEST", *v.Version, path)
+		assert.Equal(t, "prod", *v.Environment.Variables["MODE"], path)
+		v.FunctionArn = cfg.FunctionArn
+		assert.Equal(t, cfg, *v, "apart from the qualified ARN it is the configuration read")
+	}
+}
+
+func TestListVersionsByFunction_RefusesBadMaxItemsAndAnyMarker(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	for _, q := range []string{"MaxItems=0", "MaxItems=10001", "MaxItems=-1", "MaxItems=ten", "MaxItems=", "Marker=not-a-marker", "Marker="} {
+		_, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/versions?"+q, nil)
+		requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+}
+
+func TestListVersionsByFunction_MissingFunctionIsNotFound(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/ghost/versions", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, getErr := e.do(http.MethodGet, "/2015-03-31/functions/ghost", nil)
+	assert.Equal(t, getErr.Error(), err.Error())
+
+	e.h.Region = "us-west-2"
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/versions", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestListVersionsByFunction_DoesNotNeedTheCodeStore(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+	e.deps.Codes = func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") }
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/versions", nil)
+	require.NoError(t, err)
+	e.deps.Codes = nil
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello/versions", nil)
+	require.NoError(t, err)
+}
+
+func TestListVersionsByFunction_Authorization(t *testing.T) {
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	setup := func(t *testing.T, statements ...iamdomain.Statement) *env {
+		e := newEnv(t, statements...)
+		_, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+			FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+			Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+		})
+		require.NoError(t, err)
+		return e
+	}
+	list := func(e *env) error {
+		_, err := e.do(http.MethodGet, "/2015-03-31/functions/hello/versions", nil)
+		return err
+	}
+
+	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
+		gwtest.AssertDenied(t, list(setup(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))))
+	})
+	t.Run("a deny on the function ARN applies", func(t *testing.T) {
+		gwtest.AssertDenied(t, list(setup(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Deny", "lambda:ListVersionsByFunction", arn))))
+	})
+	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
+		gwtest.AssertDenied(t, list(setup(t, gwtest.Statement("Allow", "lambda:ListVersionsByFunction", arn+"-other"))))
+	})
+	t.Run("an allow on the function ARN passes", func(t *testing.T) {
+		require.NoError(t, list(setup(t, gwtest.Statement("Allow", "lambda:ListVersionsByFunction", arn))))
+	})
+}

spinifex/domains/lambda/awsapi/registration.go

@@ -117,7 +117,7 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		if decodeErr == nil {
 			resource = functionARN(inv.Region, inv.AccountID, aws.StringValue(c.Create.FunctionName))
 		}
-	case "GetFunction":
+	case "GetFunction", "GetFunctionConfiguration", "GetFunctionCodeSigningConfig", "ListVersionsByFunction":
 		if name, _, err := resolveFunctionName(inv.Region, inv.AccountID, params[0]); err == nil {
 			resource = functionARN(inv.Region, inv.AccountID, name)
 		}
@@ -133,7 +133,10 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 			return err
 		}
 	}
-	if d.Meta == nil || d.Codes == nil {
+	// Only GetFunction presigns code; the other reads are metadata only.
+	needsCodes := action != "GetFunctionConfiguration" && action != "GetFunctionCodeSigningConfig" &&
+		action != "ListVersionsByFunction"
+	if d.Meta == nil || (d.Codes == nil && needsCodes) {
 		return errors.New(awserrors.ErrorServiceException)
 	}
 
@@ -180,7 +183,13 @@ func (d Deps) checkPassRole(inv dispatch.Invocation, roleARN string) error {
 }
 
 func writeJSON(w http.ResponseWriter, out reply) {
-	body, err := jsonutil.BuildJSON(out.Body)
+	var body []byte
+	var err error
+	if raw, ok := out.Body.(rawJSON); ok {
+		body = raw
+	} else {
+		body, err = jsonutil.BuildJSON(out.Body)
+	}
 	if err != nil {
 		slog.Error("Failed to marshal Lambda response", "err", err)
 		w.Header().Set("Content-Type", "application/json")

spinifex/domains/lambda/awsapi/routes.go

@@ -21,6 +21,10 @@ type call struct {
 	Create *lambda.CreateFunctionInput
 }
 
+// rawJSON is a response body written as is, for a shape the SDK encoder
+// cannot produce.
+type rawJSON []byte
+
 // reply is a handler's success: the HTTP status and the SDK shape to encode.
 type reply struct {
 	Status int
@@ -41,6 +45,18 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.getFunction(ctx, c)
 		}},
+	{Method: "GET", Pattern: "/2015-03-31/functions/{FunctionName}/configuration", Action: "GetFunctionConfiguration",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.getFunctionConfiguration(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/functions/{FunctionName}/versions", Action: "ListVersionsByFunction",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.listVersionsByFunction(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2020-06-30/functions/{FunctionName}/code-signing-config", Action: "GetFunctionCodeSigningConfig",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.getFunctionCodeSigningConfig(ctx, c)
+		}},
 }
 
 // actionNames returns the distinct actions in routes in stable order.

spinifex/domains/sqs/storevolume/prepare.go

@@ -31,7 +31,7 @@ type VolumeService interface {
 // Publisher is the provider pair used to reach the new volume's blocks.
 type Publisher interface {
 	PublishVolume(context.Context, ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error)
-	UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) error
+	UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error)
 }
 
 // Request names the volume to create.
@@ -204,9 +204,10 @@ func (p *Preparer) fill(ctx context.Context, volumeID, image string) error {
 func (p *Preparer) unpublish(ctx context.Context, volumeID string) error {
 	cctx, cancel := p.cleanupContext(ctx)
 	defer cancel()
-	return p.Provider.UnpublishVolume(cctx, ebsprovider.UnpublishVolumeRequest{
+	_, err := p.Provider.UnpublishVolume(cctx, ebsprovider.UnpublishVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: p.NodeID,
 	})
+	return err
 }
 
 // fail discards the volume when its export is known closed and otherwise

spinifex/domains/sqs/storevolume/prepare_test.go

@@ -105,14 +105,17 @@ func (h *harness) PublishVolume(_ context.Context, req ebsprovider.PublishVolume
 	return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: export}, nil
 }
 
-func (h *harness) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error {
+func (h *harness) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
 	h.record("unpublish " + req.VolumeID + " on " + req.NodeID)
 	h.unpublishCtxOK = ctx.Err() == nil
 	if h.unpublishBlocks {
 		<-ctx.Done()
-		return ctx.Err()
+		return nil, ctx.Err()
 	}
-	return h.unpublishErr
+	if h.unpublishErr != nil {
+		return nil, h.unpublishErr
+	}
+	return &ebsprovider.UnpublishedVolume{}, nil
 }
 
 func (h *harness) preparer(t *testing.T) *Preparer {

spinifex/operator/accountlifecycle/engine.go

@@ -35,6 +35,12 @@ type Engine struct {
 	// TERMINATING and before the first stage. It records evidence and gates nothing.
 	OnQuiesced func(ctx context.Context, quiescence operatorv1.IAMQuiescence)
 
+	// Cleanup runs the EC2 platform cleanup. Only Exercise uses it.
+	Cleanup PlatformCleaner
+
+	// OnExercised receives what a non-finalizing run did after quiescence.
+	OnExercised func(ctx context.Context, result operatorv1.ExerciseResult)
+
 	// ec2Inventory is EC2's read-only platform inventory. Only DryRun consults
 	// it, and only to log a comparison; no teardown or Inventory call does.
 	ec2Inventory ec2PlatformInventory
@@ -129,7 +135,7 @@ func (e *Engine) Teardown(ctx context.Context, req Request) (*Result, error) {
 		return nil, fmt.Errorf("quiesce account: %w", err)
 	}
 	if e.OnQuiesced != nil {
-		e.OnQuiesced(ctx, e.evaluateQuiescence(ctx, req, generation))
+		e.OnQuiesced(ctx, e.evaluateQuiescence(ctx, req, generation, false))
 	}
 	e.logger().InfoContext(ctx, "Account teardown started",
 		"accountID", account.AccountID, "accountName", account.AccountName, "force", req.Force)
@@ -338,9 +344,9 @@ func SortReapers(reapers []Reaper) {
 
 // evaluateQuiescence reads back what the quiesce wrote and, only if the fence
 // is confirmed, asks EC2 for its platform inventory within a short budget.
-func (e *Engine) evaluateQuiescence(ctx context.Context, req Request, generation uint64) operatorv1.IAMQuiescence {
+func (e *Engine) evaluateQuiescence(ctx context.Context, req Request, generation uint64, exercise bool) operatorv1.IAMQuiescence {
 	ev := QuiesceEvidence{
-		OperationID: req.OperationID, Fence: req.Fence, Generation: generation, EvaluatedAt: time.Now(),
+		OperationID: req.OperationID, Fence: req.Fence, Generation: generation, EvaluatedAt: time.Now(), Exercise: exercise,
 	}
 	ev.Account, ev.ReadbackErr = e.Accounts.GetAccount(req.AccountID)
 	if _, confirmed := fenceReadback(ev); confirmed && e.ec2Inventory != nil {

spinifex/operator/accountlifecycle/exercise.go

@@ -0,0 +1,207 @@
+package accountlifecycle
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
+	"github.com/nats-io/nats.go"
+)
+
+// PlatformCleaner is the EC2 platform cleanup the exercise runs.
+type PlatformCleaner = accountcleanup.PlatformCleaner
+
+// Exercise quiesces the account, cleans the EC2 platform kinds the evaluation
+// recorded as quiesced, then re-reads the owner inventory and holds it against
+// the cleanup. No legacy stage runs and the account record is never removed.
+func (e *Engine) Exercise(ctx context.Context, req Request) (*operatorv1.ExerciseResult, error) {
+	if _, err := e.checkDeletable(req.AccountID, req.AccountName); err != nil {
+		return nil, err
+	}
+	if req.Force {
+		return nil, errors.New("a non-finalizing run refuses force")
+	}
+	if e.Cleanup == nil || e.ec2Inventory == nil {
+		return nil, errors.New("exercise needs the platform cleanup and inventory")
+	}
+	generation, err := e.Accounts.QuiesceAccount(req.AccountID, req.OperationID, req.Fence)
+	if err != nil {
+		return nil, fmt.Errorf("quiesce account: %w", err)
+	}
+	quiescence := e.evaluateQuiescence(ctx, req, generation, true)
+	if e.OnQuiesced != nil {
+		e.OnQuiesced(ctx, quiescence)
+	}
+
+	result := &operatorv1.ExerciseResult{}
+	if len(quiescence.QuiescedScope) == 0 {
+		result.Skipped = operatorv1.ExerciseSkippedNoQuiescedKind
+	} else {
+		cleaned, err := e.Cleanup.CleanupPlatform(ctx, &accountcleanup.Request{
+			AccountID: req.AccountID, CorrelationRef: req.OperationID, Kinds: quiescence.QuiescedScope,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("platform cleanup: %w", err)
+		}
+		result.Cleanup = platformCleanup(cleaned)
+		result.Accounting = e.accountForRemainder(ctx, req, quiescence.QuiescedScope, result.Cleanup)
+	}
+	if e.OnExercised != nil {
+		e.OnExercised(ctx, *result)
+	}
+	return result, nil
+}
+
+// accountForRemainder re-reads the owner inventory after cleanup and holds it
+// against the cleanup result, per quiesced kind.
+func (e *Engine) accountForRemainder(ctx context.Context, req Request, kinds []string, cleanup *operatorv1.PlatformCleanup) []operatorv1.KindAccounting {
+	budget := ec2PlatformShadowBudget
+	if e.ec2PlatformShadowBudget > 0 {
+		budget = e.ec2PlatformShadowBudget
+	}
+	rereadCtx, cancel := context.WithTimeout(ctx, budget)
+	defer cancel()
+	reply := e.ec2Inventory.PlatformInventory(rereadCtx, req.AccountID, req.OperationID)
+	return AccountForRemainder(kinds, &reply, cleanup)
+}
+
+// AccountForRemainder marks a kind accounted only when its re-read was complete
+// and every record still there is named in the cleanup result with a reason.
+// A record the cleanup reported deleted that is back is unnamed.
+func AccountForRemainder(kinds []string, reread *ec2v1.PlatformInventoryReply, cleanup *operatorv1.PlatformCleanup) []operatorv1.KindAccounting {
+	out := make([]operatorv1.KindAccounting, 0, len(kinds))
+	for _, kind := range kinds {
+		entry := operatorv1.KindAccounting{Kind: kind, Status: operatorv1.AccountingAccounted}
+		status, complete := kindInventory(reread, kind)
+		if !complete {
+			entry.Status, entry.Reason = operatorv1.AccountingUnaccounted, operatorv1.AccountingReasonRereadIncomplete
+			out = append(out, entry)
+			continue
+		}
+		if status == ec2v1.PlatformStatusRecordsFound {
+			unnamed := false
+			for _, item := range reread.Items {
+				if item.Kind != kind {
+					continue
+				}
+				entry.Remaining++
+				unnamed = unnamed || !namedItem(cleanup, item.ResourceID, kind)
+			}
+			for _, count := range reread.Counts {
+				if count.Kind != kind {
+					continue
+				}
+				entry.Remaining += count.Count
+				unnamed = unnamed || !namedCount(cleanup, count)
+			}
+			if unnamed {
+				entry.Status, entry.Reason = operatorv1.AccountingUnaccounted, operatorv1.AccountingReasonUnnamedRemainder
+			}
+		}
+		out = append(out, entry)
+	}
+	return out
+}
+
+func namedItem(cleanup *operatorv1.PlatformCleanup, id, kind string) bool {
+	if cleanup == nil {
+		return false
+	}
+	for _, item := range cleanup.Items {
+		if item.ResourceID == id && item.Kind == kind && item.Disposition != accountcleanup.DispositionDeleted && item.Reason != "" {
+			return true
+		}
+	}
+	return false
+}
+
+func namedCount(cleanup *operatorv1.PlatformCleanup, want ec2v1.PlatformRecordCount) bool {
+	if cleanup == nil {
+		return false
+	}
+	for _, count := range cleanup.Counts {
+		if count.Kind == want.Kind && count.Class == want.Class && count.Disposition != accountcleanup.DispositionDeleted &&
+			count.Reason != "" && count.Count >= want.Count {
+			return true
+		}
+	}
+	return false
+}
+
+func platformCleanup(res *accountcleanup.Result) *operatorv1.PlatformCleanup {
+	out := &operatorv1.PlatformCleanup{Kinds: []operatorv1.CleanupKind{}}
+	for _, kind := range res.Kinds {
+		out.Kinds = append(out.Kinds, operatorv1.CleanupKind{Kind: kind.Kind, Status: kind.Status, IncompleteReasons: kind.IncompleteReasons})
+	}
+	for _, item := range res.Items {
+		out.Items = append(out.Items, operatorv1.CleanupItem{
+			ResourceID: item.ResourceID, Kind: item.Kind, Disposition: item.Disposition, Reason: item.Reason, MemberCount: item.MemberCount,
+		})
+	}
+	for _, count := range res.Counts {
+		out.Counts = append(out.Counts, operatorv1.CleanupCount{
+			Kind: count.Kind, Class: count.Class, Disposition: count.Disposition, Reason: count.Reason, Count: count.Count,
+		})
+	}
+	return out
+}
+
+// InstanceReferenceReader reads the key-pair and placement-group names the
+// account's non-terminated instances carry. It fails closed: a listing that is
+// not complete from every node is an error, never a partial answer.
+type InstanceReferenceReader struct {
+	nc      *nats.Conn
+	nodeIDs []string
+	budget  time.Duration
+}
+
+var _ accountcleanup.InstanceReferenceReader = (*InstanceReferenceReader)(nil)
+
+// NewInstanceReferenceReader builds the reader over the configured node set.
+func NewInstanceReferenceReader(nc *nats.Conn, nodeIDs []string) *InstanceReferenceReader {
+	return &InstanceReferenceReader{nc: nc, nodeIDs: nodeIDs, budget: ec2PlatformShadowBudget}
+}
+
+func (r *InstanceReferenceReader) AccountInstanceReferences(ctx context.Context, accountID string) (*accountcleanup.References, error) {
+	ctx, cancel := context.WithTimeout(ctx, r.budget)
+	defer cancel()
+	out, err := ec2instanceapi.DescribeAccountInstancesComplete(ctx, r.nc, r.nodeIDs, accountID)
+	if err != nil {
+		return nil, err
+	}
+	return instanceReferences(out), nil
+}
+
+func instanceReferences(out *ec2.DescribeInstancesOutput) *accountcleanup.References {
+	refs := &accountcleanup.References{
+		KeyNames: map[string]bool{}, PlacementGroupNames: map[string]bool{}, PlacementGroupIDs: map[string]bool{},
+	}
+	for _, reservation := range out.Reservations {
+		for _, instance := range reservation.Instances {
+			if instance == nil {
+				continue
+			}
+			if instance.State != nil && instance.State.Name != nil && terminalInstanceStates[*instance.State.Name] {
+				continue
+			}
+			if instance.KeyName != nil && *instance.KeyName != "" {
+				refs.KeyNames[*instance.KeyName] = true
+			}
+			if instance.Placement != nil {
+				if instance.Placement.GroupName != nil && *instance.Placement.GroupName != "" {
+					refs.PlacementGroupNames[*instance.Placement.GroupName] = true
+				}
+				if instance.Placement.GroupId != nil && *instance.Placement.GroupId != "" {
+					refs.PlacementGroupIDs[*instance.Placement.GroupId] = true
+				}
+			}
+		}
+	}
+	return refs
+}

spinifex/operator/accountlifecycle/exercise_test.go

@@ -0,0 +1,288 @@
+package accountlifecycle
+
+//test:in-package — the exercise's inventory and cleanup collaborators are
+// unexported seams, and the S7 invariants drive the engine directly.
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type stubCleaner struct {
+	calls  []accountcleanup.Request
+	result *accountcleanup.Result
+	err    error
+	// onCall runs inside the cleanup, so a test can observe what was recorded first.
+	onCall func()
+}
+
+func (s *stubCleaner) CleanupPlatform(_ context.Context, req *accountcleanup.Request) (*accountcleanup.Result, error) {
+	s.calls = append(s.calls, *req)
+	if s.onCall != nil {
+		s.onCall()
+	}
+	return s.result, s.err
+}
+
+// sequencedInventory answers the quiescence evaluation first, then the re-read.
+type sequencedInventory struct {
+	replies []ec2v1.PlatformInventoryReply
+	calls   int
+}
+
+func (s *sequencedInventory) PlatformInventory(context.Context, string, string) ec2v1.PlatformInventoryReply {
+	reply := s.replies[min(s.calls, len(s.replies)-1)]
+	s.calls++
+	return reply
+}
+
+func exerciseEngine(accounts AccountStore, cleaner PlatformCleaner, inventory ec2PlatformInventory) *Engine {
+	engine := quietEngine(accounts)
+	engine.Cleanup = cleaner
+	engine.ec2Inventory = inventory
+	return engine
+}
+
+func exerciseRequest() Request {
+	req := request()
+	req.OperationID, req.Fence = "op-1", 1
+	return req
+}
+
+func cleanResult() *accountcleanup.Result {
+	res := &accountcleanup.Result{AccountID: testAccountID}
+	for _, kind := range ec2v1.PlatformKinds() {
+		res.Kinds = append(res.Kinds, accountcleanup.KindReport{Kind: kind, Status: ec2v1.PlatformStatusObservedClean})
+	}
+	return res
+}
+
+func accountingFor(result *operatorv1.ExerciseResult, kind string) operatorv1.KindAccounting {
+	for _, entry := range result.Accounting {
+		if entry.Kind == kind {
+			return entry
+		}
+	}
+	return operatorv1.KindAccounting{}
+}
+
+// The exercise records the evaluation before it cleans, cleans only the kinds that
+// evaluation recorded as quiesced, and never removes the account record.
+func TestExerciseCleansTheQuiescedKindsAfterRecordingAndKeepsTheAccount(t *testing.T) {
+	accounts := newFakeAccounts()
+	var recorded *operatorv1.IAMQuiescence
+	cleaner := &stubCleaner{result: cleanResult()}
+	cleaner.onCall = func() { assert.NotNil(t, recorded, "quiescence must be recorded before any cleanup") }
+	inventory := &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory()}}
+	engine := exerciseEngine(accounts, cleaner, inventory)
+	engine.OnQuiesced = func(_ context.Context, q operatorv1.IAMQuiescence) { recorded = &q }
+
+	result, err := engine.Exercise(context.Background(), exerciseRequest())
+
+	require.NoError(t, err)
+	require.NotNil(t, recorded)
+	assert.ElementsMatch(t, ec2PlatformKinds, recorded.QuiescedScope)
+	require.Len(t, cleaner.calls, 1)
+	assert.ElementsMatch(t, recorded.QuiescedScope, cleaner.calls[0].Kinds)
+	assert.Equal(t, "op-1", cleaner.calls[0].CorrelationRef)
+	assert.False(t, accounts.deleted, "an exercise never removes the account record")
+	assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
+	assert.Equal(t, 2, inventory.calls, "one evaluation read and one final re-read")
+	for _, kind := range recorded.QuiescedScope {
+		assert.Equal(t, operatorv1.AccountingAccounted, accountingFor(result, kind).Status, kind)
+	}
+}
+
+// The final re-read is held against the cleanup result and the quiescence record
+// is not rewritten from it.
+func TestExerciseAccountsForARemainderOnlyWhenTheCleanupNamedIt(t *testing.T) {
+	cases := map[string]struct {
+		cleanup *accountcleanup.Result
+		reread  ec2v1.PlatformInventoryReply
+		want    string
+		reason  string
+	}{
+		"late write the cleanup never saw": {
+			cleanup: cleanResult(),
+			reread:  rereadWithKeyPair("key-late"),
+			want:    operatorv1.AccountingUnaccounted, reason: operatorv1.AccountingReasonUnnamedRemainder,
+		},
+		"cleanup deleted it but it is back": {
+			cleanup: withItem(cleanResult(), accountcleanup.Item{ResourceID: "key-1", Kind: ec2v1.PlatformKindKeyPair, Disposition: accountcleanup.DispositionDeleted}),
+			reread:  rereadWithKeyPair("key-1"),
+			want:    operatorv1.AccountingUnaccounted, reason: operatorv1.AccountingReasonUnnamedRemainder,
+		},
+		"cleanup blocked it with a reason": {
+			cleanup: withItem(cleanResult(), accountcleanup.Item{ResourceID: "key-1", Kind: ec2v1.PlatformKindKeyPair, Disposition: accountcleanup.DispositionBlocked, Reason: accountcleanup.ReasonInstanceReference}),
+			reread:  rereadWithKeyPair("key-1"),
+			want:    operatorv1.AccountingAccounted,
+		},
+		"re-read incomplete": {
+			cleanup: cleanResult(),
+			reread:  *incompleteKeyPairs(),
+			want:    operatorv1.AccountingUnaccounted, reason: operatorv1.AccountingReasonRereadIncomplete,
+		},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			accounts := newFakeAccounts()
+			inventory := &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory(), tc.reread}}
+			engine := exerciseEngine(accounts, &stubCleaner{result: tc.cleanup}, inventory)
+			var recorded operatorv1.IAMQuiescence
+			engine.OnQuiesced = func(_ context.Context, q operatorv1.IAMQuiescence) { recorded = q }
+
+			result, err := engine.Exercise(context.Background(), exerciseRequest())
+
+			require.NoError(t, err)
+			entry := accountingFor(result, ec2v1.PlatformKindKeyPair)
+			assert.Equal(t, tc.want, entry.Status)
+			assert.Equal(t, tc.reason, entry.Reason)
+			assert.Contains(t, recorded.QuiescedScope, ec2v1.PlatformKindKeyPair, "quiescence is not rewritten from the re-read")
+		})
+	}
+}
+
+func rereadWithKeyPair(id string) ec2v1.PlatformInventoryReply {
+	reply := *completeInventory()
+	reply.Kinds[0].Status = ec2v1.PlatformStatusRecordsFound
+	reply.Items = []ec2v1.PlatformItem{{ResourceID: id, Kind: ec2v1.PlatformKindKeyPair, Disposition: accountcleanup.DispositionBlocked}}
+	return reply
+}
+
+func incompleteKeyPairs() *ec2v1.PlatformInventoryReply {
+	reply := completeInventory()
+	reply.Kinds[0].Status = ec2v1.PlatformStatusScanIncomplete
+	return reply
+}
+
+func withItem(res *accountcleanup.Result, item accountcleanup.Item) *accountcleanup.Result {
+	res.Items = append(res.Items, item)
+	return res
+}
+
+// With no kind quiesced the exercise runs no cleanup and records why.
+func TestExerciseWithNoQuiescedKindRunsNoCleanup(t *testing.T) {
+	accounts := newFakeAccounts()
+	cleaner := &stubCleaner{result: cleanResult()}
+	incomplete := ec2v1.IncompletePlatformInventory(testAccountID, "op-1", ec2v1.PlatformReasonTimeout)
+	engine := exerciseEngine(accounts, cleaner, &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{incomplete}})
+
+	result, err := engine.Exercise(context.Background(), exerciseRequest())
+
+	require.NoError(t, err)
+	assert.Empty(t, cleaner.calls)
+	assert.Equal(t, operatorv1.ExerciseSkippedNoQuiescedKind, result.Skipped)
+	assert.Nil(t, result.Cleanup)
+}
+
+func TestExerciseRefusesForceAndAMissingCollaborator(t *testing.T) {
+	accounts := newFakeAccounts()
+	req := exerciseRequest()
+	req.Force = true
+	_, err := exerciseEngine(accounts, &stubCleaner{}, &stubInventory{}).Exercise(context.Background(), req)
+	require.Error(t, err)
+
+	_, err = exerciseEngine(accounts, nil, &stubInventory{}).Exercise(context.Background(), exerciseRequest())
+	require.Error(t, err)
+	assert.Empty(t, accounts.quiesces, "a refused exercise quiesces nothing")
+}
+
+// ADR-0003:S7: the exercise is an automated actor that deletes customer-visible
+// records, so it acts only on kinds whose quiescence was recorded under a confirmed
+// fence, and a failed cleanup leaves the account where it is.
+func TestS7_ExerciseCleansNothingUnlessItsQuiescenceWasRecordedUnderAConfirmedFence(t *testing.T) {
+	cases := map[string]func(*Engine, *fakeAccounts) *sequencedInventory{
+		"read-back fails": func(e *Engine, a *fakeAccounts) *sequencedInventory {
+			e.Accounts = &readbackFails{fakeAccounts: a}
+			return &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory()}}
+		},
+		"inventory incomplete": func(*Engine, *fakeAccounts) *sequencedInventory {
+			return &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{
+				ec2v1.IncompletePlatformInventory(testAccountID, "op-1", ec2v1.PlatformReasonNoResponder),
+			}}
+		},
+	}
+	for name, setup := range cases {
+		t.Run(name, func(t *testing.T) {
+			accounts := newFakeAccounts()
+			cleaner := &stubCleaner{result: cleanResult()}
+			engine := exerciseEngine(accounts, cleaner, nil)
+			engine.ec2Inventory = setup(engine, accounts)
+
+			_, err := engine.Exercise(context.Background(), exerciseRequest())
+
+			require.NoError(t, err)
+			assert.Empty(t, cleaner.calls, "no kind may be cleaned without a recorded quiescence")
+			assert.False(t, accounts.deleted)
+		})
+	}
+
+	t.Run("only the kinds with a complete inventory are cleaned", func(t *testing.T) {
+		accounts := newFakeAccounts()
+		cleaner := &stubCleaner{result: cleanResult()}
+		partial := completeInventory()
+		partial.Kinds[1].Status = ec2v1.PlatformStatusScanIncomplete
+		engine := exerciseEngine(accounts, cleaner, &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*partial}})
+
+		_, err := engine.Exercise(context.Background(), exerciseRequest())
+
+		require.NoError(t, err)
+		require.Len(t, cleaner.calls, 1)
+		assert.ElementsMatch(t, []string{ec2v1.PlatformKindKeyPair, ec2v1.PlatformKindLaunchTemplate}, cleaner.calls[0].Kinds)
+	})
+
+	t.Run("a failed cleanup returns an error and removes nothing", func(t *testing.T) {
+		accounts := newFakeAccounts()
+		engine := exerciseEngine(accounts, &stubCleaner{err: errors.New("owner down")}, &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory()}})
+
+		_, err := engine.Exercise(context.Background(), exerciseRequest())
+
+		require.Error(t, err)
+		assert.False(t, accounts.deleted)
+	})
+}
+
+// ADR-0003:S7: an ordinary teardown never runs the platform cleanup and never
+// records a quiesced scope, whatever its inventory says.
+func TestS7_OrdinaryTeardownNeverCleansOrRecordsAScope(t *testing.T) {
+	accounts := newFakeAccounts()
+	cleaner := &stubCleaner{result: cleanResult()}
+	engine := exerciseEngine(accounts, cleaner, &sequencedInventory{replies: []ec2v1.PlatformInventoryReply{*completeInventory()}})
+	var recorded operatorv1.IAMQuiescence
+	engine.OnQuiesced = func(_ context.Context, q operatorv1.IAMQuiescence) { recorded = q }
+
+	_, err := engine.Teardown(context.Background(), exerciseRequest())
+
+	require.NoError(t, err)
+	assert.Empty(t, cleaner.calls)
+	assert.Empty(t, recorded.QuiescedScope)
+}
+
+func TestInstanceReferencesSkipTerminalInstancesAndNameKeysAndGroups(t *testing.T) {
+	refs := instanceReferences(&ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{
+		instanceWith("running", "kp-a", "pg-a", "pg-1"),
+		instanceWith("stopped", "kp-b", "", ""),
+		instanceWith("terminated", "kp-gone", "pg-gone", "pg-9"),
+	}}}})
+
+	assert.Equal(t, map[string]bool{"kp-a": true, "kp-b": true}, refs.KeyNames)
+	assert.Equal(t, map[string]bool{"pg-a": true}, refs.PlacementGroupNames)
+	assert.Equal(t, map[string]bool{"pg-1": true}, refs.PlacementGroupIDs)
+}
+
+func instanceWith(state, key, groupName, groupID string) *ec2.Instance {
+	return &ec2.Instance{
+		State:     &ec2.InstanceState{Name: aws.String(state)},
+		KeyName:   aws.String(key),
+		Placement: &ec2.Placement{GroupName: aws.String(groupName), GroupId: aws.String(groupID)},
+	}
+}

spinifex/operator/accountlifecycle/quiescence.go

@@ -7,17 +7,17 @@ import (
 	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
 )
 
-// finalInventoryRereadWired says whether anything re-reads the owner inventory
-// before the account record goes. Until it does, a write landing after the
-// reaper stage is orphaned, so no kind may be recorded as quiesced.
-var finalInventoryRereadWired = false
-
 // QuiesceEvidence is what the engine read after QuiesceAccount returned.
 type QuiesceEvidence struct {
 	OperationID string
 	Fence       uint64
 	Generation  uint64
 
+	// Exercise is set only for a non-finalizing run, which re-reads the owner
+	// inventory after its cleanup and removes no account record. An ordinary
+	// delete re-reads nothing before the record goes, so it records no scope.
+	Exercise bool
+
 	// Account is the IAM record read back after the quiesce.
 	Account     *Account
 	ReadbackErr error
@@ -79,7 +79,8 @@ var otherScopes = []unresolvedEntry{
 
 // EvaluateQuiescence applies ADR-0008:S9's three conditions to the evidence and
 // records what holds. A fence the read-back does not confirm stops everything
-// else; otherwise every scope is listed, and none is reported quiesced yet.
+// else; otherwise every scope is listed. A kind is quiesced only for an
+// exercise whose owner inventory for it was complete.
 func EvaluateQuiescence(ev QuiesceEvidence) operatorv1.IAMQuiescence {
 	out := operatorv1.IAMQuiescence{
 		OperationID:   ev.OperationID,
@@ -109,7 +110,7 @@ func EvaluateQuiescence(ev QuiesceEvidence) operatorv1.IAMQuiescence {
 		switch {
 		case !complete:
 			out.Unresolved = append(out.Unresolved, writerEntry("ec2-"+kind, operatorv1.ReasonOwnerInventoryIncomplete))
-		case !finalInventoryRereadWired:
+		case !ev.Exercise:
 			out.Unresolved = append(out.Unresolved, writerEntry("ec2-"+kind, operatorv1.ReasonFinalInventoryRereadNotWired))
 		default:
 			out.QuiescedScope = append(out.QuiescedScope, kind)

spinifex/operator/accountlifecycle/quiescence_test.go

@@ -72,12 +72,10 @@ func TestEvaluateQuiescenceRecordsEvidenceButQuiescesNothingWithoutAFinalReread(
 	assert.Equal(t, "2026-11-02T00:00:00Z", q.EvaluatedAt)
 }
 
-// The wiring step flips this switch; the complete kinds are then, and only then, named.
+// Only an exercise, which re-reads the inventory after cleanup, names the complete kinds.
 func TestEvaluateQuiescenceNamesCompleteKindsOnceTheRereadIsWired(t *testing.T) {
-	finalInventoryRereadWired = true
-	t.Cleanup(func() { finalInventoryRereadWired = false })
-
 	ev := confirmedEvidence()
+	ev.Exercise = true
 	ev.Inventory.Kinds[1].Status = ec2v1.PlatformStatusScanIncomplete
 	q := EvaluateQuiescence(ev)
 
@@ -133,9 +131,8 @@ func TestEvaluateQuiescenceRecordsNothingWhenTheFenceIsNotConfirmed(t *testing.T
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
-			finalInventoryRereadWired = true
-			t.Cleanup(func() { finalInventoryRereadWired = false })
 			ev := confirmedEvidence()
+			ev.Exercise = true
 			tc.mutate(&ev)
 			q := EvaluateQuiescence(ev)
 
@@ -172,11 +169,11 @@ func TestEveryOtherScopeIsRecordedUnresolvedByName(t *testing.T) {
 
 // Predastore has no accepted enforcement contract, so the scopes it admits writes
 // for stay unresolved and it is never cited as a consumer, even once the
-// re-read is wired and every L09 kind is complete.
+// run is an exercise and every L09 kind is complete.
 func TestPredastoreIsNeverCitedAndItsScopesNeverQuiesce(t *testing.T) {
-	finalInventoryRereadWired = true
-	t.Cleanup(func() { finalInventoryRereadWired = false })
-	q := EvaluateQuiescence(confirmedEvidence())
+	ev := confirmedEvidence()
+	ev.Exercise = true
+	q := EvaluateQuiescence(ev)
 
 	for _, scope := range []string{"s3", "ecr"} {
 		got := unresolvedFor(q, scope)

spinifex/operator/admin/account_delete.go

@@ -66,7 +66,10 @@ type accountDeletionJob struct {
 	ClientToken string `json:"clientToken"`
 	State       string `json:"state"`
 	Force       bool   `json:"force"`
-	StartedAt   string `json:"startedAt"`
+	// NonFinalizing marks an exercise: it quiesces and cleans the quiesced EC2
+	// platform kinds, then ends EXERCISED with the account still TERMINATING.
+	NonFinalizing bool   `json:"nonFinalizing,omitempty"`
+	StartedAt     string `json:"startedAt"`
 	// UpdatedAt is the running job's heartbeat. Its absence for long enough is
 	// what lets another gateway take over after a restart.
 	UpdatedAt  string                        `json:"updatedAt"`
@@ -88,6 +91,8 @@ type accountDeletionJob struct {
 	// IAM is the quiescence evaluation made under the current fence. A takeover
 	// starts without one and the new holder records its own.
 	IAM *operatorv1.IAMQuiescence `json:"iam,omitempty"`
+	// Exercise is what a non-finalizing run did after quiescence.
+	Exercise *operatorv1.ExerciseResult `json:"exercise,omitempty"`
 }
 
 // deletionTakeover records one change of holder.
@@ -99,8 +104,9 @@ type deletionTakeover struct {
 }
 
 const (
-	takeoverReasonLeaseExpired = "lease_expired"
-	takeoverReasonFailed       = "failed"
+	takeoverReasonLeaseExpired     = "lease_expired"
+	takeoverReasonFailed           = "failed"
+	takeoverReasonExerciseFinished = "exercise_finished"
 )
 
 // deleteAccount starts a teardown, or answers a dry run inline.
@@ -118,6 +124,11 @@ func (d Deps) deleteAccount(ctx context.Context, body []byte) (any, error) {
 		return nil, errors.New(awserrors.ErrorInternalError)
 	}
 
+	if req.NonFinalizing && d.PlatformCleaner == nil {
+		slog.Error("DeleteAccount: platform cleanup not available")
+		return nil, errors.New(awserrors.ErrorServiceUnavailable)
+	}
+
 	engine, err := accountteardown.NewClusterEngine(ctx, d.NATS, accountteardown.ClusterNodes{Expected: d.ExpectedNodes, DaemonIDs: d.NodeIDs, GatewayIDs: d.GatewayNodeIDs}, d.Termination, d.BucketStore)
 	if err != nil {
 		slog.Error("DeleteAccount: failed to build teardown engine", "err", err)
@@ -262,6 +273,10 @@ func (d Deps) runAccountDeletion(
 	tracker := &deletionTracker{jobs: jobs, job: job, revision: revision, cancel: cancel}
 	engine.OnStage = tracker.stageFinished
 	engine.OnQuiesced = tracker.quiesced
+	if job.NonFinalizing {
+		engine.Cleanup = d.PlatformCleaner
+		engine.OnExercised = tracker.exercised
+	}
 
 	go func() {
 		defer cancel()
@@ -269,13 +284,19 @@ func (d Deps) runAccountDeletion(
 		stop := tracker.startHeartbeat(ctx)
 		defer stop()
 
-		result, err := engine.Teardown(ctx, accountteardown.Request{
+		request := accountteardown.Request{
 			AccountID:   job.AccountID,
 			AccountName: job.AccountName,
 			Force:       job.Force,
 			OperationID: job.OperationID,
 			Fence:       job.Fence,
-		})
+		}
+		if job.NonFinalizing {
+			_, err := engine.Exercise(ctx, request)
+			tracker.finished(ctx, nil, err)
+			return
+		}
+		result, err := engine.Teardown(ctx, request)
 		tracker.finished(ctx, result, err)
 	}()
 }
@@ -308,6 +329,13 @@ func (t *deletionTracker) quiesced(ctx context.Context, quiescence operatorv1.IA
 	t.persist(ctx)
 }
 
+func (t *deletionTracker) exercised(ctx context.Context, result operatorv1.ExerciseResult) {
+	t.mu.Lock()
+	t.job.Exercise = &result
+	t.mu.Unlock()
+	t.persist(ctx)
+}
+
 func (t *deletionTracker) finished(ctx context.Context, result *accountteardown.Result, err error) {
 	t.mu.Lock()
 	if result != nil {
@@ -316,6 +344,8 @@ func (t *deletionTracker) finished(ctx context.Context, result *accountteardown.
 	if err != nil {
 		t.job.State = operatorv1.DeletionStateFailed
 		t.job.Error = err.Error()
+	} else if t.job.NonFinalizing {
+		t.job.State = operatorv1.DeletionStateExercised
 	} else {
 		t.job.State = operatorv1.DeletionStateCompleted
 	}
@@ -445,8 +475,10 @@ func claimDeletionJob(
 		StartedAt:   now.Format(time.RFC3339),
 		UpdatedAt:   now.Format(time.RFC3339),
 		Fence:       1,
-		Holder:      holder,
-		LeaseUntil:  now.Add(accountDeletionStale).Format(time.RFC3339),
+
+		NonFinalizing: req.NonFinalizing,
+		Holder:        holder,
+		LeaseUntil:    now.Add(accountDeletionStale).Format(time.RFC3339),
 	}
 	record, err := json.Marshal(job)
 	if err != nil {
@@ -478,6 +510,9 @@ func claimDeletionJob(
 	}
 
 	reason := takeoverReasonFailed
+	if existing.State == operatorv1.DeletionStateExercised {
+		reason = takeoverReasonExerciseFinished
+	}
 	if existing.State == operatorv1.DeletionStateRunning {
 		if !deletionLeaseExpired(existing, now) {
 			return nil, 0, nil, errors.New(awserrors.ErrorOperationInProgress)
@@ -512,8 +547,10 @@ func takeOverDeletionJob(
 		Force:       req.Force,
 		StartedAt:   now.Format(time.RFC3339),
 		UpdatedAt:   now.Format(time.RFC3339),
-		Holder:      holder,
-		LeaseUntil:  now.Add(accountDeletionStale).Format(time.RFC3339),
+
+		NonFinalizing: req.NonFinalizing,
+		Holder:        holder,
+		LeaseUntil:    now.Add(accountDeletionStale).Format(time.RFC3339),
 	}
 	if existing.OperationID != "" {
 		job.OperationID = existing.OperationID
@@ -557,7 +594,7 @@ func replayDeletionJob(
 		return nil, err
 	}
 	if existing.ClientToken == req.ClientToken {
-		if existing.AccountName != req.AccountName {
+		if existing.AccountName != req.AccountName || existing.NonFinalizing != req.NonFinalizing {
 			return nil, errors.New(awserrors.ErrorIdempotentParameterMismatch)
 		}
 		// A RUNNING job whose lease has lapsed has no holder to follow, so the
@@ -703,6 +740,9 @@ func validateDeleteAccountRequest(req *operatorv1.DeleteAccountRequest) error {
 	if !accountIDRE.MatchString(req.AccountID) {
 		return errors.New(awserrors.ErrorInvalidParameterValue)
 	}
+	if req.NonFinalizing && (req.Force || req.DryRun) {
+		return errors.New(awserrors.ErrorInvalidParameterCombination)
+	}
 	if req.DryRun {
 		// A dry run deletes nothing, so it needs neither the name confirmation
 		// nor a token to make it idempotent.

spinifex/operator/admin/account_delete_exercise_test.go

@@ -0,0 +1,133 @@
+package admin
+
+//test:in-package — the exercise runs through the job record, the takeover and
+// the resumer, all of which are unexported.
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const exerciseToken = "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d"
+
+type panicCleaner struct{ calls int }
+
+func (p *panicCleaner) CleanupPlatform(context.Context, *accountcleanup.Request) (*accountcleanup.Result, error) {
+	p.calls++
+	return &accountcleanup.Result{}, nil
+}
+
+func exerciseBody(t *testing.T, accountID, name, token string, mutate func(*operatorv1.DeleteAccountRequest)) []byte {
+	t.Helper()
+	req := operatorv1.DeleteAccountRequest{AccountID: accountID, AccountName: name, ClientToken: token, NonFinalizing: true}
+	if mutate != nil {
+		mutate(&req)
+	}
+	body, err := json.Marshal(req)
+	require.NoError(t, err)
+	return body
+}
+
+func awaitState(t *testing.T, gw Deps, accountID, state string) *accountDeletionJob {
+	t.Helper()
+	require.Eventually(t, func() bool {
+		return describeDeletion(t, gw, accountID).State == state
+	}, 30*time.Second, 100*time.Millisecond)
+	return describeDeletion(t, gw, accountID)
+}
+
+// An exercise ends EXERCISED with the account still TERMINATING, and a later
+// ordinary delete takes the same operation over to finish it.
+func TestNonFinalizingDeleteEndsExercisedAndALaterDeleteTakesItOver(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	gw.PlatformCleaner = &panicCleaner{}
+	account := newTenantAccount(t, gw, "tenant@example.com")
+
+	_, err := gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken, nil))
+	require.NoError(t, err)
+
+	exercised := awaitState(t, gw, account.AccountID, operatorv1.DeletionStateExercised)
+	require.NotNil(t, exercised.IAM)
+	require.NotNil(t, exercised.Exercise)
+	assert.True(t, exercised.NonFinalizing)
+	assert.NotEmpty(t, exercised.FinishedAt)
+	assert.Empty(t, exercised.Stages, "no legacy stage runs")
+	stored, err := gw.IAM.GetAccount(account.AccountID)
+	require.NoError(t, err)
+	assert.Equal(t, "TERMINATING", stored.Status)
+
+	_, err = gw.deleteAccount(t.Context(), deleteAccountBody(t, account.AccountID, "tenant@example.com", deleteClientToken))
+	require.NoError(t, err)
+
+	done := awaitState(t, gw, account.AccountID, operatorv1.DeletionStateCompleted)
+	assert.Equal(t, exercised.OperationID, done.OperationID)
+	assert.Equal(t, uint64(2), done.Fence)
+	require.Len(t, done.Takeovers, 1)
+	assert.Equal(t, takeoverReasonExerciseFinished, done.Takeovers[0].Reason)
+	assert.Nil(t, done.Exercise, "the takeover starts a fresh run")
+	assert.Equal(t, uint64(2), done.IAM.Fence)
+	assert.Empty(t, done.IAM.QuiescedScope, "an ordinary delete records no scope")
+}
+
+func TestNonFinalizingIsPartOfTheIdempotencyKeyAndRefusesForce(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	gw.PlatformCleaner = &panicCleaner{}
+	account := newTenantAccount(t, gw, "tenant@example.com")
+
+	_, err := gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken,
+		func(r *operatorv1.DeleteAccountRequest) { r.Force = true }))
+	require.EqualError(t, err, awserrors.ErrorInvalidParameterCombination)
+
+	_, err = gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken, nil))
+	require.NoError(t, err)
+	awaitState(t, gw, account.AccountID, operatorv1.DeletionStateExercised)
+
+	_, err = gw.deleteAccount(t.Context(), deleteAccountBody(t, account.AccountID, "tenant@example.com", exerciseToken))
+	require.EqualError(t, err, awserrors.ErrorIdempotentParameterMismatch, "the same token cannot turn an exercise into a delete")
+
+	again, err := gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken, nil))
+	require.NoError(t, err)
+	assert.Equal(t, operatorv1.DeletionStateExercised, again.(*DeleteAccountResponse).State, "a repeated exercise is answered from the record")
+}
+
+func TestNonFinalizingWithoutACleanupIsRefusedBeforeAnyJobExists(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	account := newTenantAccount(t, gw, "tenant@example.com")
+
+	_, err := gw.deleteAccount(t.Context(), exerciseBody(t, account.AccountID, "tenant@example.com", exerciseToken, nil))
+
+	require.EqualError(t, err, awserrors.ErrorServiceUnavailable)
+	jobs, err := gw.accountDeletionStore(t.Context())
+	require.NoError(t, err)
+	stored, err := readDeletionJob(t.Context(), jobs, account.AccountID)
+	require.NoError(t, err)
+	assert.Nil(t, stored)
+}
+
+// A resumed exercise stays an exercise: the resumer must not turn a recorded
+// non-finalizing decision into a deletion.
+func TestResumerKeepsAnAbandonedExerciseNonFinalizing(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	gw.PlatformCleaner = &panicCleaner{}
+	account := newTenantAccount(t, gw, "tenant@example.com")
+	job := abandonedJob(account.AccountID, "tenant@example.com")
+	job.NonFinalizing = true
+	storeDeletionJob(t, gw, job)
+
+	require.Equal(t, 1, gw.resumeDeletions(t.Context(), time.Now().UTC()))
+
+	done := awaitState(t, gw, account.AccountID, operatorv1.DeletionStateExercised)
+	assert.Equal(t, uint64(2), done.Fence)
+	assert.True(t, done.NonFinalizing)
+	stored, err := gw.IAM.GetAccount(account.AccountID)
+	require.NoError(t, err)
+	assert.Equal(t, "TERMINATING", stored.Status, "the account record is never removed")
+}

spinifex/operator/admin/account_delete_resumer.go

@@ -110,6 +110,8 @@ func (d Deps) resumeDeletion(
 		AccountName: existing.AccountName,
 		ClientToken: existing.ClientToken,
 		Force:       existing.Force,
+
+		NonFinalizing: existing.NonFinalizing,
 	}
 	job, newRevision, _, err := takeOverDeletionJob(ctx, jobs, req, d.holderID(), existing, revision, takeoverReasonLeaseExpired, now)
 	if err != nil {

spinifex/operator/admin/account_delete_resumer_test.go

@@ -45,6 +45,7 @@ func TestS7_DeletionResumerActsOnlyOnAnAcceptedOperationWithoutALiveLease(t *tes
 		},
 		"failed job":    func(j *accountDeletionJob) { j.State = operatorv1.DeletionStateFailed },
 		"completed job": func(j *accountDeletionJob) { j.State = operatorv1.DeletionStateCompleted },
+		"exercised job": func(j *accountDeletionJob) { j.State = operatorv1.DeletionStateExercised },
 		"legacy record": func(j *accountDeletionJob) { j.OperationID, j.Fence, j.LeaseUntil = "", 0, "" },
 	}
 	for name, mutate := range cases {

spinifex/operator/admin/image_import.go

@@ -76,7 +76,7 @@ func ImportImage(ctx context.Context, provider ebsprovider.EBSProvider, opts Imp
 
 	// Unpublish regardless: a volume left published cannot be snapshotted or
 	// attached, and on the failure path it would strand the export too.
-	if err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
+	if _, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(),
 		VolumeID:  opts.VolumeID,
 		NodeID:    opts.NodeID,

spinifex/operator/admin/image_import_test.go

@@ -49,7 +49,7 @@ func (p *recordingProvider) PublishVolume(ctx context.Context, req ebsprovider.P
 	return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: export}, nil
 }
 
-func (p *recordingProvider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error {
+func (p *recordingProvider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
 	p.calls = append(p.calls, "unpublish")
 	return p.MemoryProvider.UnpublishVolume(ctx, req)
 }

spinifex/operator/admin/registration.go

@@ -58,7 +58,10 @@ type Deps struct {
 	IAM iamdomain.IAMService
 	// Termination is the concrete IAM service, the only holder of the capability
 	// to quiesce and remove an account. DeleteAccount needs it.
-	Termination       accountteardown.IAMBackend
+	Termination accountteardown.IAMBackend
+	// PlatformCleaner is the in-process EC2 platform cleanup a non-finalizing
+	// DeleteAccount runs. Nil refuses that request.
+	PlatformCleaner   accountteardown.PlatformCleaner
 	Quota             *quota.Service
 	ExpectedNodes     int
 	NodeIDs           []string

spinifex/operator/cli/admin_account_delete.go

@@ -61,6 +61,8 @@ func init() {
 	accountCmd.AddCommand(accountDeleteCmd)
 
 	accountDeleteCmd.Flags().Bool("dry-run", false, "Report what would be deleted and change nothing")
+	accountDeleteCmd.Flags().Bool("non-finalizing", false, "Quiesce and clean the EC2 platform kinds, then stop with the account still TERMINATING")
+	_ = accountDeleteCmd.Flags().MarkHidden("non-finalizing")
 	accountDeleteCmd.Flags().Bool("force", false, "Escalate past state guards for resources that will not delete")
 	accountDeleteCmd.Flags().Bool("yes", false, "Skip the interactive confirmation (requires --name)")
 	accountDeleteCmd.Flags().String("name", "", "Account name, which must match the stored record")

spinifex/operator/cli/admin_account_delete_remote.go

@@ -24,6 +24,7 @@ var accountDeletePollInterval = 10 * time.Second
 func runAccountDeleteRemote(cmd *cobra.Command, accountID string) error {
 	dryRun, _ := cmd.Flags().GetBool("dry-run")
 	force, _ := cmd.Flags().GetBool("force")
+	nonFinalizing, _ := cmd.Flags().GetBool("non-finalizing")
 	assumeYes, _ := cmd.Flags().GetBool("yes")
 	name, _ := cmd.Flags().GetString("name")
 	clientToken, _ := cmd.Flags().GetString("client-token")
@@ -66,6 +67,7 @@ func runAccountDeleteRemote(cmd *cobra.Command, accountID string) error {
 
 	started, err := deleteAccountRemote(ctx, target, operatorv1.DeleteAccountRequest{
 		AccountID: accountID, AccountName: name, ClientToken: clientToken, Force: force,
+		NonFinalizing: nonFinalizing,
 	})
 	if err != nil {
 		var adminErr *adminError
@@ -101,6 +103,9 @@ func followAccountDeletion(ctx context.Context, target adminTarget, accountID st
 		case operatorv1.DeletionStateCompleted:
 			fmt.Printf("\nAccount %s deleted.\n", accountID)
 			return nil
+		case operatorv1.DeletionStateExercised:
+			fmt.Printf("\nAccount %s exercised and left TERMINATING; a normal delete finishes it.\n", accountID)
+			return nil
 		case operatorv1.DeletionStateFailed:
 			fmt.Fprintf(os.Stderr, "\nThe account is still TERMINATING so its leftovers keep an owner.\n")
 			return fmt.Errorf("teardown failed: %s", job.Error)

spinifex/operator/cli/admin_test.go

@@ -1748,3 +1748,31 @@ func TestApplyNetworkConfig_PropagatesPoolDNSServersWithoutSelf(t *testing.T) {
 	require.Len(t, settings.Pools, 1)
 	assert.Equal(t, []string{"1.1.1.1", "10.0.0.6", "8.8.8.8"}, settings.Pools[0].DNSServers)
 }
+
+// A generated config writes no [nodes.<peer>.ebs] stanza, so a node switched to
+// a non-default provider must still load: an absent peer provider is unknown.
+func TestSpinifexTomlTemplate_NonDefaultProviderLoadsBesidePeers(t *testing.T) {
+	allNodes := map[string]formation.NodeInfo{
+		"node1": {Name: "node1", BindIP: "10.0.0.1", AdvertiseIP: "192.168.1.31"},
+		"node2": {Name: "node2", BindIP: "10.0.0.2", AdvertiseIP: "192.168.1.32"},
+		"node3": {Name: "node3", BindIP: "10.0.0.3", AdvertiseIP: "192.168.1.33"},
+	}
+	settings := multinodeNorthstarSettings("/etc/spinifex/northstar/northstar.toml")
+	settings.RemoteNodes = buildRemoteNodes(allNodes, "node1", settings.NorthstarConfigPath)
+	require.Len(t, settings.RemoteNodes, 2)
+
+	path := filepath.Join(t.TempDir(), "spinifex.toml")
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+	rendered, err := os.ReadFile(path)
+	require.NoError(t, err)
+	const defaultLine = "provider = \"viperblockd\"\n"
+	require.Equal(t, 1, strings.Count(string(rendered), defaultLine), "the template must set the provider for its own node only")
+	switched := strings.Replace(string(rendered), defaultLine, "provider = \"qemunbd\"\n", 1)
+	require.NoError(t, os.WriteFile(path, []byte(switched), 0o600))
+
+	cfg, err := config.LoadConfig(path)
+	require.NoError(t, err)
+	assert.Equal(t, config.EBSProviderQEMUNBD, cfg.Nodes["node1"].EBS.ResolvedProvider())
+	assert.Contains(t, cfg.Nodes, "node2")
+	assert.Contains(t, cfg.Nodes, "node3")
+}

spinifex/operator/cli/templates/spinifex.toml

@@ -136,6 +136,9 @@ encryption_key_file = "{{.EncryptionKeyFile}}"
 {{- end}}
 
 [nodes.{{.Node}}.ebs]
+# provider: every node in a cluster must name the same one, and the daemon
+# refuses to start on a provider other than the one the cluster recorded.
+# Changing it needs the cluster wiped and reformed.
 provider = "viperblockd"
 # Per-volume export tunables. Both are applied per nbdkit process and there is
 # one of those per volume, so the host cost of each is the value times the

spinifex/operator/upgrade/VERSIONS.md

@@ -21,6 +21,7 @@ Single source of truth for the current schema version of every config file Spini
 |---|---|---|
 | `spinifex-instance-state` | `5` | `daemon.InstanceStateBucketVersion` |
 | `spinifex-cluster-state` | `1` | `daemon.ClusterStateBucketVersion` |
+| `spinifex-cluster-config` | `1` | `daemon.ClusterConfigBucketVersion` |
 | `spinifex-terminated-instances` | `3` | `daemon.TerminatedInstanceBucketVersion` |
 
 ## Registered migrations

spinifex/providers/ebs/conformance/bench.go

@@ -137,7 +137,7 @@ func RunBenchSuite(b *testing.B, provider ebsprovider.EBSProvider, cfg BenchConf
 			}); err != nil {
 				b.Fatal(err)
 			}
-			if err := provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{
+			if _, err := provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{
 				Versioned: ebsprovider.NewVersioned(),
 				VolumeID:  volumeID,
 				NodeID:    cfg.node(),

spinifex/providers/ebs/conformance/conformance.go

@@ -6,6 +6,9 @@ package conformance
 import (
 	"bytes"
 	"context"
+	"errors"
+	"fmt"
+	"slices"
 	"strings"
 	"sync"
 	"testing"
@@ -34,18 +37,24 @@ var ReferenceCapabilities = ebsprovider.Capabilities{
 
 // capabilitiesOf reads what a provider advertises, so the suite can branch on
 // it the way the contract tells callers to.
-func capabilitiesOf(t *testing.T, provider ebsprovider.EBSProvider) ebsprovider.Capabilities {
+func capabilitiesOf(t *testing.T, provider ebsprovider.EBSProvider) *ebsprovider.GetCapabilitiesResponse {
 	t.Helper()
 	resp, err := provider.GetCapabilities(context.Background(), ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()})
 	require.NoError(t, err)
 	require.NotNil(t, resp)
-	return resp.Capabilities
+	return resp
 }
 
 // maxListPages bounds a drain so a provider whose token fails to advance
 // fails the test instead of hanging it.
 const maxListPages = 100
 
+// unpublish is UnpublishVolume for a case that asserts only on the error.
+func unpublish(ctx context.Context, provider ebsprovider.EBSProvider, req ebsprovider.UnpublishVolumeRequest) error {
+	_, err := provider.UnpublishVolume(ctx, req)
+	return err
+}
+
 // volumeIDs projects one page of refs down to its IDs.
 func volumeIDs(refs []ebsprovider.VolumeRef) []string {
 	ids := make([]string, 0, len(refs))
@@ -131,6 +140,11 @@ type SuiteConfig struct {
 	// also be published elsewhere. Empty skips that subtest, for a caller with
 	// only one node to offer.
 	OtherNodeID string
+
+	// KillExport ends the process serving a published export without telling
+	// the provider, standing in for a crash. Nil skips the dead-export case,
+	// for a provider whose export is not a separate process.
+	KillExport func(t *testing.T, published *ebsprovider.PublishedVolume)
 }
 
 func (c SuiteConfig) node() string {
@@ -176,7 +190,8 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 		return newTrackingProvider(t, newRawProvider(t))
 	}
 
-	capabilities := capabilitiesOf(t, newProvider(t))
+	advertised := capabilitiesOf(t, newProvider(t))
+	capabilities, contractVersions := advertised.Capabilities, advertised.ContractVersions
 
 	t.Run("GetCapabilities", func(t *testing.T) {
 		provider := newProvider(t)
@@ -193,6 +208,44 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 		assert.Equal(t, resp.Capabilities, again.Capabilities, "GetCapabilities must be stable across calls")
 	})
 
+	t.Run("ContractVersions", func(t *testing.T) {
+		t.Run("listed versions are ascending and include the legacy version", func(t *testing.T) {
+			require.NotEmpty(t, contractVersions, "a caller cannot choose a version for a provider that lists none")
+			assert.Contains(t, contractVersions, ebsprovider.LegacySchemaVersion,
+				"dropping the legacy version strands every caller still in an N/N-1 rollout")
+			for i := 1; i < len(contractVersions); i++ {
+				assert.Lessf(t, contractVersions[i-1], contractVersions[i], "contract versions %v are not strictly ascending", contractVersions)
+			}
+		})
+
+		t.Run("every listed version is answered in that version", func(t *testing.T) {
+			provider := newProvider(t)
+			for _, version := range contractVersions {
+				resp, err := provider.GetCapabilities(context.Background(), ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}})
+				require.NoErrorf(t, err, "version %d is listed as served", version)
+				assert.Equalf(t, version, resp.SchemaVersion, "a reply in another version than the request's is one an older caller may not parse")
+			}
+		})
+
+		t.Run("an unlisted version is refused without changing state", func(t *testing.T) {
+			provider := newProvider(t)
+			ctx := context.Background()
+			unknown := ebsprovider.Versioned{SchemaVersion: ebsprovider.SchemaVersion + 1}
+			_, err := provider.GetCapabilities(ctx, ebsprovider.GetCapabilitiesRequest{Versioned: unknown})
+			require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion)
+
+			volumeID := cfg.id("vol-ver-unknown")
+			_, err = provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: unknown, VolumeID: volumeID, NodeID: cfg.node()})
+			require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion)
+			// A refused publish that still exported the volume would leave it
+			// published, and a published volume cannot be deleted.
+			require.NoError(t, provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID}),
+				"a publish refused for its version must not have published the volume")
+		})
+	})
+
 	t.Run("CreateVolume", func(t *testing.T) {
 		t.Run("success", func(t *testing.T) {
 			provider := newProvider(t)
@@ -385,8 +438,6 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 	})
 
 	t.Run("ListVolumes", func(t *testing.T) {
-		capabilities := capabilitiesOf(t, newProvider(t))
-
 		t.Run("unsupported_capability when enumeration is not advertised", func(t *testing.T) {
 			if capabilities.VolumeEnumeration {
 				t.Skip("provider advertises volume enumeration")
@@ -694,8 +745,6 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 	})
 
 	t.Run("ListSnapshots", func(t *testing.T) {
-		capabilities := capabilitiesOf(t, newProvider(t))
-
 		t.Run("unsupported_capability when enumeration is not advertised", func(t *testing.T) {
 			if capabilities.SnapshotEnumeration {
 				t.Skip("provider advertises snapshot enumeration")
@@ -798,16 +847,23 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			}
 		})
 
+		// A republish succeeds at every listed version, so a caller at any of
+		// them gets the running export back rather than a refusal.
 		t.Run("idempotent republish to the same node", func(t *testing.T) {
-			provider := newProvider(t)
-			ctx := context.Background()
-			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-idem"), CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
-			require.NoError(t, err)
-			first, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-idem"), NodeID: cfg.node()})
-			require.NoError(t, err)
-			second, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-pub-idem"), NodeID: cfg.node()})
-			require.NoError(t, err)
-			assert.Equal(t, first, second)
+			for _, version := range contractVersions {
+				t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) {
+					provider := newProvider(t)
+					ctx := context.Background()
+					volumeID := cfg.id(fmt.Sprintf("vol-pub-idem-v%d", version))
+					_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+					require.NoError(t, err)
+					first, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+					require.NoError(t, err)
+					second, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: volumeID, NodeID: cfg.node()})
+					require.NoError(t, err)
+					assert.Equal(t, first, second)
+				})
+			}
 		})
 
 		t.Run("volume_in_use when published to a different node", func(t *testing.T) {
@@ -824,6 +880,76 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse)
 		})
 
+		t.Run("held elsewhere is reported only from the version that defines it", func(t *testing.T) {
+			if cfg.otherNode() == "" {
+				t.Skip("no second node configured")
+			}
+			for _, version := range contractVersions {
+				t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) {
+					provider := newProvider(t)
+					ctx := context.Background()
+					volumeID := cfg.id(fmt.Sprintf("vol-pub-held-v%d", version))
+					_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+					require.NoError(t, err)
+					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+					require.NoError(t, err)
+
+					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: volumeID, NodeID: cfg.otherNode()})
+					require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse, "every version must still read the refusal as in use")
+					if version >= ebsprovider.HeldElsewhereSchemaVersion {
+						assert.ErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere)
+					} else {
+						assert.NotErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere,
+							"a version %d caller predates held_elsewhere and must get the volume_in_use it has always got", version)
+					}
+				})
+			}
+		})
+
+		t.Run("a republish in the other access mode is in use, not held elsewhere", func(t *testing.T) {
+			if !capabilities.ReadOnlyPublish {
+				t.Skip("read-only publish is not advertised")
+			}
+			for _, version := range contractVersions {
+				t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) {
+					provider := newProvider(t)
+					ctx := context.Background()
+					volumeID := cfg.id(fmt.Sprintf("vol-pub-mode-v%d", version))
+					_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+					require.NoError(t, err)
+					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+					require.NoError(t, err)
+
+					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: volumeID, NodeID: cfg.node(), ReadOnly: true})
+					require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse)
+					assert.NotErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere,
+						"this node holds the volume, so a caller told it is held elsewhere would wait for a node that does not exist")
+				})
+			}
+		})
+
+		t.Run("a republish after the export died returns a serving export", func(t *testing.T) {
+			if cfg.KillExport == nil {
+				t.Skip("no way to kill this provider's export")
+			}
+			RequireNBDTools(t)
+			provider := newProvider(t)
+			ctx := context.Background()
+			volumeID := cfg.id("vol-pub-dead")
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			first, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+			firstURI := dialURI(t, publishedNBDURI(t, first))
+
+			cfg.KillExport(t, first)
+			require.False(t, nbdReachable(firstURI), "the export still answers, so the kill did not take and this case proves nothing")
+
+			second, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err, "a dead export must not make its volume unpublishable")
+			nbdInfoRetry(t, dialURI(t, publishedNBDURI(t, second)))
+		})
+
 		t.Run("not_found on absent volume", func(t *testing.T) {
 			provider := newProvider(t)
 			_, err := provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed"), NodeID: cfg.node()})
@@ -845,21 +971,105 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			require.NoError(t, err)
 			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-unpub-ok"), NodeID: cfg.node()})
 			require.NoError(t, err)
-			require.NoError(t, provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-unpub-ok"), NodeID: cfg.node()}))
+			require.NoError(t, unpublish(ctx, provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-unpub-ok"), NodeID: cfg.node()}))
 		})
 
 		t.Run("absent target is idempotent", func(t *testing.T) {
 			provider := newProvider(t)
-			require.NoError(t, provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed"), NodeID: cfg.node()}))
+			require.NoError(t, unpublish(context.Background(), provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed"), NodeID: cfg.node()}))
+		})
+
+		t.Run("torn down reports whether an export was removed", func(t *testing.T) {
+			if !slices.Contains(contractVersions, ebsprovider.NoSealSchemaVersion) {
+				t.Skipf("torn_down is defined from version %d, which the provider does not list", ebsprovider.NoSealSchemaVersion)
+			}
+			provider := newProvider(t)
+			ctx := context.Background()
+			volumeID := cfg.id("vol-unpub-torn")
+			v3 := ebsprovider.Versioned{SchemaVersion: ebsprovider.NoSealSchemaVersion}
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+
+			first, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: v3, VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+			assert.True(t, first.TornDown, "the volume was published, so this unpublish removed an export")
+			again, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: v3, VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+			assert.False(t, again.TornDown, "nothing was left to remove, so a retry must not claim it removed something")
+		})
+
+		t.Run("no_seal below the version that defines it is refused", func(t *testing.T) {
+			for _, version := range contractVersions {
+				if version >= ebsprovider.NoSealSchemaVersion {
+					continue
+				}
+				t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) {
+					provider := newProvider(t)
+					ctx := context.Background()
+					volumeID := cfg.id(fmt.Sprintf("vol-unpub-noseal-v%d", version))
+					_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+					require.NoError(t, err)
+					_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+					require.NoError(t, err)
+
+					err = unpublish(ctx, provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: volumeID, NodeID: cfg.node(), NoSeal: true})
+					require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument,
+						"a version %d server would ignore no_seal and seal, so the flag must be refused rather than dropped", version)
+					err = provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID})
+					require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse, "the refused unpublish must have left the volume published")
+				})
+			}
+		})
+
+		t.Run("no_seal is honoured or refused as unsupported", func(t *testing.T) {
+			provider := newProvider(t)
+			ctx := context.Background()
+			volumeID := cfg.id("vol-unpub-noseal")
+			v3 := ebsprovider.Versioned{SchemaVersion: ebsprovider.NoSealSchemaVersion}
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+
+			result, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: v3, VolumeID: volumeID, NodeID: cfg.node(), NoSeal: true})
+			if errors.Is(err, ebsprovider.ErrUnsupportedVersion) {
+				return
+			}
+			require.NoError(t, err)
+			assert.True(t, result.TornDown, "the volume was published, so a no_seal unpublish removed an export")
+			again, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: v3, VolumeID: volumeID, NodeID: cfg.node(), NoSeal: true})
+			require.NoError(t, err)
+			assert.False(t, again.TornDown)
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err, "a volume dropped without sealing must still be publishable")
 		})
 
 		t.Run("invalid_argument on missing ids", func(t *testing.T) {
 			provider := newProvider(t)
-			err := provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned()})
+			err := unpublish(context.Background(), provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned()})
 			require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
 		})
 	})
 
+	t.Run("HoldsUnsealedWrites", func(t *testing.T) {
+		t.Run("answers with a subset of the volumes asked about", func(t *testing.T) {
+			provider := newProvider(t)
+			ctx := context.Background()
+			volumeID := cfg.id("vol-unsealed")
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+
+			asked := []string{volumeID, cfg.id("vol-unsealed-never")}
+			held, err := provider.HoldsUnsealedWrites(ctx, cfg.node(), asked)
+			require.NoError(t, err, "an error reads as cannot say, which blocks every start elsewhere")
+			assert.Subset(t, asked, held, "a volume nobody asked about is not an answer to the question")
+		})
+	})
+
 	t.Run("Exclusion", func(t *testing.T) {
 		exclusion := capabilities.Exclusion
 
@@ -907,7 +1117,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 
 			// Releasing the claim must make the volume publishable again, or
 			// the guarantee is indistinguishable from a permanent lock.
-			require.NoError(t, provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()}))
+			require.NoError(t, unpublish(ctx, provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()}))
 			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.otherNode()})
 			require.NoError(t, err, "a released claim must be reclaimable, or an unpublish strands the volume")
 		})
@@ -968,7 +1178,7 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse,
 				"deleting a published volume must be refused; the export is still serving reads and writes against it")
 
-			require.NoError(t, provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()}))
+			require.NoError(t, unpublish(ctx, provider, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()}))
 			require.NoError(t, provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID}),
 				"once unpublished the delete must go through, or a volume can never be removed after being used")
 		})

spinifex/providers/ebs/conformance/nbdclient.go

@@ -217,7 +217,7 @@ func cleanupVolume(t *testing.T, provider ebsprovider.EBSProvider, cfg NBDClient
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
-	if err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
+	if _, err := provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.NodeID,
 	}); err != nil {
 		t.Logf("cleanup: unpublish %s: %v", volumeID, err)
@@ -310,7 +310,7 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
 	// holes to a server that really tracks allocation.
 	t.Run("sparse extent reporting matches the advertised capability", func(t *testing.T) {
 		provider := newProvider(t)
-		capabilities := capabilitiesOf(t, provider)
+		capabilities := capabilitiesOf(t, provider).Capabilities
 		nbdURI := publishForNBD(t, provider, cfg, cfg.VolumePrefix+"-extents", false)
 
 		holes := slices.ContainsFunc(nbdMap(t, dialURI(t, nbdURI)), func(extent nbdExtent) bool {
@@ -323,7 +323,7 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
 
 	t.Run("read-only publish matches the advertised capability", func(t *testing.T) {
 		provider := newProvider(t)
-		capabilities := capabilitiesOf(t, provider)
+		capabilities := capabilitiesOf(t, provider).Capabilities
 
 		if !capabilities.ReadOnlyPublish {
 			// Refusing is the contract for a provider that cannot honour it;
@@ -356,11 +356,12 @@ func RunNBDClientSuiteWithConfig(t *testing.T, newProvider func(t *testing.T) eb
 		uri := dialURI(t, nbdURI)
 		require.True(t, nbdReachable(uri), "export should be reachable before unpublish")
 
-		require.NoError(t, provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{
+		_, err := provider.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{
 			Versioned: ebsprovider.NewVersioned(),
 			VolumeID:  volumeID,
 			NodeID:    cfg.NodeID,
-		}))
+		})
+		require.NoError(t, err)
 
 		assert.Eventually(t, func() bool { return !nbdReachable(uri) }, 30*time.Second, 250*time.Millisecond,
 			"export still answers after UnpublishVolume")

spinifex/providers/ebs/conformance/tracking.go

@@ -84,12 +84,12 @@ func (p *trackingProvider) PublishVolume(ctx context.Context, req ebsprovider.Pu
 	return pub, err
 }
 
-func (p *trackingProvider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error {
-	err := p.EBSProvider.UnpublishVolume(ctx, req)
+func (p *trackingProvider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
+	result, err := p.EBSProvider.UnpublishVolume(ctx, req)
 	if err == nil {
 		delete(p.published, req.VolumeID)
 	}
-	return err
+	return result, err
 }
 
 // cleanup removes what the subtest left behind, in dependency order: a
@@ -100,7 +100,7 @@ func (p *trackingProvider) cleanup() {
 	ctx := context.Background()
 
 	for volumeID, nodeID := range p.published {
-		if err := p.EBSProvider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
+		if _, err := p.EBSProvider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
 			Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, Handle: p.volumes[volumeID], NodeID: nodeID,
 		}); err != nil {
 			p.t.Logf("conformance cleanup: unpublish %s: %v", volumeID, err)

spinifex/providers/ebs/errors.go

@@ -12,6 +12,11 @@ var (
 	ErrUnsupportedVersion = errors.New("unsupported EBS provider schema version")
 	ErrVolumeInUse        = errors.New("volume is in use")
 
+	// ErrVolumeHeldElsewhere is a publish refused because another node holds
+	// the volume's claim. It wraps ErrVolumeInUse, so a caller that only asks
+	// whether the volume is in use still matches it.
+	ErrVolumeHeldElsewhere = fmt.Errorf("volume is held by another node: %w", ErrVolumeInUse)
+
 	// ErrUnsupportedCapability is returned when a request asks for optional
 	// behaviour this provider does not advertise in Capabilities. It is
 	// distinct from ErrInvalidArgument: the request is well formed.
@@ -28,19 +33,35 @@ var (
 	ErrUnavailable = errors.New("provider is temporarily unable to answer")
 )
 
+// HeldElsewhereSchemaVersion is the first version a held-elsewhere refusal is
+// reported as such. Earlier requests get volume_in_use, as v1.23.0 sends it.
+const HeldElsewhereSchemaVersion uint16 = 3
+
+// HeldElsewhereError is the sentinel a held-elsewhere refusal of a request at
+// version wraps: ErrVolumeHeldElsewhere from HeldElsewhereSchemaVersion on.
+func HeldElsewhereError(version uint16) error {
+	if version >= HeldElsewhereSchemaVersion {
+		return ErrVolumeHeldElsewhere
+	}
+	return ErrVolumeInUse
+}
+
 // ErrorCode is the wire form of a provider error in ProviderError. Unwrap maps
 // every code but internal back to the matching Err sentinel.
 type ErrorCode string
 
 const (
-	ErrorCodeAlreadyExists      ErrorCode = "already_exists"
-	ErrorCodeInvalidArgument    ErrorCode = "invalid_argument"
-	ErrorCodeNotFound           ErrorCode = "not_found"
-	ErrorCodeUnsupportedVersion ErrorCode = "unsupported_version"
-	ErrorCodeVolumeInUse        ErrorCode = "volume_in_use"
-	ErrorCodeUnsupportedCap     ErrorCode = "unsupported_capability"
-	ErrorCodeUnavailable        ErrorCode = "unavailable"
-	ErrorCodeInternal           ErrorCode = "internal"
+	ErrorCodeAlreadyExists       ErrorCode = "already_exists"
+	ErrorCodeInvalidArgument     ErrorCode = "invalid_argument"
+	ErrorCodeNotFound            ErrorCode = "not_found"
+	ErrorCodeUnsupportedVersion  ErrorCode = "unsupported_version"
+	ErrorCodeVolumeInUse         ErrorCode = "volume_in_use"
+	ErrorCodeVolumeHeldElsewhere ErrorCode = "volume_held_elsewhere"
+	ErrorCodeUnsupportedCap      ErrorCode = "unsupported_capability"
+	// ErrorCodeUnavailable is the retry-later code: among others, a mount
+	// against a backing store that is not ready yet returns it.
+	ErrorCodeUnavailable ErrorCode = "unavailable"
+	ErrorCodeInternal    ErrorCode = "internal"
 )
 
 // Retryable reports whether repeating the identical request could succeed.
@@ -78,6 +99,8 @@ func (e *ProviderError) Unwrap() error {
 		return ErrUnsupportedVersion
 	case ErrorCodeVolumeInUse:
 		return ErrVolumeInUse
+	case ErrorCodeVolumeHeldElsewhere:
+		return ErrVolumeHeldElsewhere
 	case ErrorCodeUnsupportedCap:
 		return ErrUnsupportedCapability
 	case ErrorCodeUnavailable:
@@ -89,7 +112,12 @@ func (e *ProviderError) Unwrap() error {
 
 func checkVersion(version uint16) error {
 	if !SupportsSchemaVersion(version) {
-		return fmt.Errorf("%w: got %d, supported versions are %d and %d", ErrUnsupportedVersion, version, LegacySchemaVersion, SchemaVersion)
+		return fmt.Errorf("%w: got %d, %s", ErrUnsupportedVersion, version, SupportedVersionsText())
 	}
 	return nil
 }
+
+// SupportedVersionsText names the served version range for error messages.
+func SupportedVersionsText() string {
+	return fmt.Sprintf("supported versions are %d to %d", LegacySchemaVersion, SchemaVersion)
+}

spinifex/providers/ebs/hostile/soak_test.go

@@ -105,7 +105,7 @@ func soakOneVolume(ctx context.Context, provider ebsprovider.EBSProvider, volume
 	if _, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: "node-1",
 	}); err == nil {
-		_ = provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
+		_, _ = provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
 			Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: "node-1",
 		})
 	}

spinifex/providers/ebs/hostile/verbs.go

@@ -21,6 +21,7 @@ const (
 	verbSnapshotDelete  = "snapshot.delete"
 	verbSnapshotCopy    = "snapshot.copy"
 	verbSnapshotList    = "snapshot.list"
+	verbUnsealedWrites  = "volume.unsealed"
 )
 
 // lieFlavours are the ways a successful answer can be wrong. Each one is a
@@ -240,10 +241,28 @@ func (p *Provider) PublishVolume(ctx context.Context, req ebsprovider.PublishVol
 	return published, nil
 }
 
-func (p *Provider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) error {
+func (p *Provider) UnpublishVolume(ctx context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
 	injection := p.draw(verbVolumeUnpublish, req.VolumeID)
 	if stop, err := p.apply(ctx, injection); stop || err != nil {
-		return err
+		return nil, err
 	}
-	return p.after(injection, p.inner.UnpublishVolume(ctx, req))
+	result, err := p.inner.UnpublishVolume(ctx, req)
+	if err := p.after(injection, err); err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+// HoldsUnsealedWrites is never made to lie: a false "none held" lets an
+// instance start elsewhere over writes only this node has.
+func (p *Provider) HoldsUnsealedWrites(ctx context.Context, nodeID string, volumeIDs []string) ([]string, error) {
+	injection := p.drawTruthful(verbUnsealedWrites, nodeID)
+	if stop, err := p.apply(ctx, injection); stop || err != nil {
+		return nil, err
+	}
+	held, err := p.inner.HoldsUnsealedWrites(ctx, nodeID, volumeIDs)
+	if err := p.after(injection, err); err != nil {
+		return nil, err
+	}
+	return held, nil
 }

spinifex/providers/ebs/memory.go

@@ -27,6 +27,7 @@ type memoryVolume struct {
 	parameters       []byte
 	seed             []byte
 	published        *PublishedVolume
+	readOnly         bool
 }
 
 var _ EBSProvider = (*MemoryProvider)(nil)
@@ -53,7 +54,11 @@ func (m *MemoryProvider) GetCapabilities(_ context.Context, req GetCapabilitiesR
 	if err := checkVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
-	return &GetCapabilitiesResponse{Versioned: NewVersioned(), Capabilities: m.capabilities}, nil
+	return &GetCapabilitiesResponse{
+		Versioned:        ReplyVersioned(req.SchemaVersion),
+		Capabilities:     m.capabilities,
+		ContractVersions: SupportedSchemaVersions(),
+	}, nil
 }
 
 func (m *MemoryProvider) CreateVolume(_ context.Context, req CreateVolumeRequest) (*Volume, error) {
@@ -167,7 +172,7 @@ func (m *MemoryProvider) ListVolumes(_ context.Context, req ListVolumesRequest)
 	defer m.mu.RUnlock()
 	ids := slices.Sorted(maps.Keys(m.volumes))
 
-	response := &ListVolumesResponse{Versioned: NewVersioned()}
+	response := &ListVolumesResponse{Versioned: ReplyVersioned(req.SchemaVersion)}
 	response.Volumes, response.NextToken = Page(ids, req.StartingToken, int(req.PageSize()),
 		func(id string) VolumeRef {
 			return VolumeRef{ID: id, Handle: m.volumes[id].volume.Handle}
@@ -338,7 +343,7 @@ func (m *MemoryProvider) ListSnapshots(_ context.Context, req ListSnapshotsReque
 	defer m.mu.RUnlock()
 	ids := slices.Sorted(maps.Keys(m.snapshots))
 
-	response := &ListSnapshotsResponse{Versioned: NewVersioned()}
+	response := &ListSnapshotsResponse{Versioned: ReplyVersioned(req.SchemaVersion)}
 	response.Snapshots, response.NextToken = Page(ids, req.StartingToken, int(req.PageSize()),
 		func(id string) SnapshotRef {
 			return SnapshotRef{
@@ -368,7 +373,10 @@ func (m *MemoryProvider) PublishVolume(_ context.Context, req PublishVolumeReque
 	}
 	if volume.published != nil {
 		if volume.published.NodeID != req.NodeID {
-			return nil, fmt.Errorf("%w: volume %s is published to %s", ErrVolumeInUse, req.VolumeID, volume.published.NodeID)
+			return nil, fmt.Errorf("%w: volume %s is published to %s", HeldElsewhereError(req.SchemaVersion), req.VolumeID, volume.published.NodeID)
+		}
+		if volume.readOnly != req.ReadOnly {
+			return nil, fmt.Errorf("%w: volume %s is already published read_only=%t", ErrVolumeInUse, req.VolumeID, volume.readOnly)
 		}
 		return clonePublished(volume.published), nil
 	}
@@ -377,26 +385,44 @@ func (m *MemoryProvider) PublishVolume(_ context.Context, req PublishVolumeReque
 		NodeID:   req.NodeID,
 		Export:   NewNBDExport("nbd+unix:///?socket=/memory/" + req.VolumeID + ".sock"),
 	}
+	volume.readOnly = req.ReadOnly
 	volume.volume.State = VolumeStateInUse
 	return clonePublished(volume.published), nil
 }
 
-func (m *MemoryProvider) UnpublishVolume(_ context.Context, req UnpublishVolumeRequest) error {
-	if err := checkVersion(req.SchemaVersion); err != nil {
-		return err
+// UnpublishVolume honours NoSeal trivially: the volume's bytes live in this
+// process, so there is no local copy to seal or to drop.
+func (m *MemoryProvider) UnpublishVolume(_ context.Context, req UnpublishVolumeRequest) (*UnpublishedVolume, error) {
+	if err := CheckUnpublishVersion(req); err != nil {
+		return nil, err
 	}
 	if req.VolumeID == "" || req.NodeID == "" {
-		return fmt.Errorf("%w: volume and node IDs are required", ErrInvalidArgument)
+		return nil, fmt.Errorf("%w: volume and node IDs are required", ErrInvalidArgument)
 	}
 	m.mu.Lock()
 	defer m.mu.Unlock()
+	result := &UnpublishedVolume{}
 	volume := m.volumes[req.VolumeID]
 	if volume == nil || volume.published == nil || volume.published.NodeID != req.NodeID {
-		return nil
+		return result, nil
 	}
 	volume.published = nil
+	volume.readOnly = false
 	volume.volume.State = VolumeStateAvailable
-	return nil
+	result.TornDown = req.SchemaVersion >= NoSealSchemaVersion
+	return result, nil
+}
+
+// HoldsUnsealedWrites answers that none are held: every write lands in this
+// process's map, which is the backend, before it is acknowledged.
+func (m *MemoryProvider) HoldsUnsealedWrites(_ context.Context, nodeID string, volumeIDs []string) ([]string, error) {
+	if nodeID == "" {
+		return nil, fmt.Errorf("%w: node ID is required", ErrInvalidArgument)
+	}
+	if slices.Contains(volumeIDs, "") {
+		return nil, fmt.Errorf("%w: empty volume ID", ErrInvalidArgument)
+	}
+	return nil, nil
 }
 
 func cloneVolume(volume *Volume) *Volume {

spinifex/providers/ebs/memory_test.go

@@ -56,9 +56,10 @@ func TestMemoryProviderLifecycleAndIdempotency(t *testing.T) {
 		Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle,
 	}), ErrVolumeInUse)
 
-	require.NoError(t, provider.UnpublishVolume(ctx, UnpublishVolumeRequest{
+	_, err = provider.UnpublishVolume(ctx, UnpublishVolumeRequest{
 		Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle, NodeID: "node-1",
-	}))
+	})
+	require.NoError(t, err)
 
 	expanded, err := provider.ExpandVolume(ctx, ExpandVolumeRequest{
 		Versioned: NewVersioned(), VolumeID: volume.ID, Handle: volume.Handle,
@@ -141,7 +142,7 @@ func TestMemoryProviderChecksVersionOnEveryMethod(t *testing.T) {
 	_, err = provider.PublishVolume(ctx, PublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"})
 	require.ErrorIs(t, err, ErrUnsupportedVersion)
 
-	err = provider.UnpublishVolume(ctx, UnpublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"})
+	_, err = provider.UnpublishVolume(ctx, UnpublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"})
 	require.ErrorIs(t, err, ErrUnsupportedVersion)
 }
 

spinifex/providers/ebs/nats.go

@@ -39,20 +39,22 @@ const defaultRequestTimeout = 30 * time.Second
 // codebase's existing budget for an EBS operation the caller cannot bound.
 const dataMovementTimeout = 180 * time.Second
 
-// dataMovementSubjects names every subject that references or moves real data.
-// Everything absent answers from state already held and keeps the caller's
-// timeout, so a subject added without a deliberate decision stays fast.
-var dataMovementSubjects = map[string]struct{}{
-	CreateVolumeSubject:   {},
-	ExpandVolumeSubject:   {},
-	CopySnapshotSubject:   {},
-	DeleteVolumeSubject:   {},
-	DeleteSnapshotSubject: {},
+// dataMovementVerbs names every operation that references or moves real data,
+// however its subject addresses it. Everything absent answers from state
+// already held and keeps the caller's timeout, so a new verb stays fast.
+var dataMovementVerbs = map[string]bool{
+	verbVolumeCreate:   true,
+	verbVolumeExpand:   true,
+	verbVolumeDelete:   true,
+	verbSnapshotCreate: true,
+	verbSnapshotCopy:   true,
+	verbSnapshotDelete: true,
+	// Unpublish seals local state back to the backend, which takes as long as
+	// the volume has unsealed writes.
+	verbVolumeUnpublish: true,
 }
 
-// timeoutFor sizes a request to the work its subject names. The per-volume
-// snapshot create and the owner-routed variants carry no constant, so they are
-// matched by prefix and by verb.
+// timeoutFor sizes a request to the work its subject names.
 func (p *NATSProvider) timeoutFor(subject string) time.Duration {
 	if movesData(subject) {
 		return max(p.requestTimeout, dataMovementTimeout)
@@ -60,28 +62,8 @@ func (p *NATSProvider) timeoutFor(subject string) time.Duration {
 	return p.requestTimeout
 }
 
-// movesData answers for the named subjects, for the per-volume snapshot create
-// prefix, and for the owner-routed variants, none of which carry a constant.
 func movesData(subject string) bool {
-	if _, ok := dataMovementSubjects[subject]; ok {
-		return true
-	}
-	if strings.HasPrefix(subject, SnapshotCreateSubjectPrefix) {
-		return true
-	}
-	_, verb, ok := ParseOwnerSubject(subject)
-	return ok && isDataMovementVerb(verb)
-}
-
-// isDataMovementVerb answers for the owner-routed subjects. volume.describe is
-// the one owner verb that reads state rather than moving data.
-func isDataMovementVerb(verb string) bool {
-	switch verb {
-	case verbSnapshotCreate, verbSnapshotCopy, verbVolumeExpand:
-		return true
-	default:
-		return false
-	}
+	return dataMovementVerbs[parseSubject(subject).verb]
 }
 
 // SnapshotSubject addresses a create by source volume so the owning node can
@@ -104,32 +86,65 @@ func SnapshotCompletionSubject(snapshotID string) (string, error) {
 	return "ebs.provider.v1.snapshot.response." + snapshotID, nil
 }
 
-// PublishSubject returns nodeID's per-node mount subject
-// (ebs.provider.v1.<node>.mount), which serves PublishVolume on that node.
-func PublishSubject(nodeID string) (string, error) {
+// Last tokens of the per-node subjects: ebs.provider.v1.<node>.<verb>.
+const (
+	publishVerb          = "mount"
+	unpublishVerb        = "unmount"
+	unsealedVerb         = "unsealed"
+	nodeCapabilitiesVerb = "capabilities"
+	fenceNoticeVerb      = "fenced"
+)
+
+// nodeSubjectWildcard is the subject matching verb on every node, for a server
+// that stands in for every node's provider.
+func nodeSubjectWildcard(verb string) string { return subjectPrefix + "*." + verb }
+
+// Wildcards over every node for each per-node subject.
+var (
+	PublishSubjectWildcard          = nodeSubjectWildcard(publishVerb)
+	UnpublishSubjectWildcard        = nodeSubjectWildcard(unpublishVerb)
+	UnsealedWritesSubjectWildcard   = nodeSubjectWildcard(unsealedVerb)
+	NodeCapabilitiesSubjectWildcard = nodeSubjectWildcard(nodeCapabilitiesVerb)
+)
+
+func nodeSubject(nodeID, verb string) (string, error) {
 	if err := validateSubjectToken(nodeID); err != nil {
 		return "", err
 	}
-	return subjectPrefix + nodeID + ".mount", nil
+	return subjectPrefix + nodeID + "." + verb, nil
+}
+
+// PublishSubject returns nodeID's per-node mount subject
+// (ebs.provider.v1.<node>.mount), which serves PublishVolume on that node.
+func PublishSubject(nodeID string) (string, error) {
+	return nodeSubject(nodeID, publishVerb)
 }
 
 // UnpublishSubject returns nodeID's per-node unmount subject
 // (ebs.provider.v1.<node>.unmount), which serves UnpublishVolume on that node.
 func UnpublishSubject(nodeID string) (string, error) {
-	if err := validateSubjectToken(nodeID); err != nil {
-		return "", err
-	}
-	return subjectPrefix + nodeID + ".unmount", nil
+	return nodeSubject(nodeID, unpublishVerb)
 }
 
 // UnsealedWritesSubject returns nodeID's per-node subject
 // (ebs.provider.v1.<node>.unsealed), which answers whether that node holds
 // unsealed writes for a set of volumes.
 func UnsealedWritesSubject(nodeID string) (string, error) {
-	if err := validateSubjectToken(nodeID); err != nil {
-		return "", err
-	}
-	return subjectPrefix + nodeID + ".unsealed", nil
+	return nodeSubject(nodeID, unsealedVerb)
+}
+
+// NodeCapabilitiesSubject returns nodeID's per-node GetCapabilities subject
+// (ebs.provider.v1.<node>.capabilities). Unlike CapabilitiesSubject, which
+// any node's provider answers, only nodeID's provider answers it.
+func NodeCapabilitiesSubject(nodeID string) (string, error) {
+	return nodeSubject(nodeID, nodeCapabilitiesVerb)
+}
+
+// FenceNoticeSubject returns nodeID's fence-notice subject
+// (ebs.provider.v1.<node>.fenced), on which a provider that revoked a live
+// export there sends a FenceNotice for that node's daemon to acknowledge.
+func FenceNoticeSubject(nodeID string) (string, error) {
+	return nodeSubject(nodeID, fenceNoticeVerb)
 }
 
 // ownerSubjectPrefix is common to every owner-routed subject OwnerSubject
@@ -224,11 +239,26 @@ func NewNATSProvider(conn *nats.Conn, requestTimeout time.Duration) *NATSProvide
 }
 
 func (p *NATSProvider) GetCapabilities(ctx context.Context, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
+	return p.getCapabilities(ctx, CapabilitiesSubject, req)
+}
+
+// GetNodeCapabilities asks nodeID's own provider what it supports, over
+// NodeCapabilitiesSubject. A provider that predates that subject gets no
+// responder, which a caller must count as no evidence.
+func (p *NATSProvider) GetNodeCapabilities(ctx context.Context, nodeID string, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
+	subject, err := NodeCapabilitiesSubject(nodeID)
+	if err != nil {
+		return nil, err
+	}
+	return p.getCapabilities(ctx, subject, req)
+}
+
+func (p *NATSProvider) getCapabilities(ctx context.Context, subject string, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
 	if err := checkVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	var response GetCapabilitiesResponse
-	if err := p.request(ctx, CapabilitiesSubject, req, &response); err != nil {
+	if err := p.request(ctx, subject, req, &response); err != nil {
 		return nil, err
 	}
 	if err := responseError(response.SchemaVersion, response.Error); err != nil {
@@ -490,19 +520,22 @@ func (p *NATSProvider) PublishVolume(ctx context.Context, req PublishVolumeReque
 	return response.Published, nil
 }
 
-func (p *NATSProvider) UnpublishVolume(ctx context.Context, req UnpublishVolumeRequest) error {
-	if err := checkVersion(req.SchemaVersion); err != nil {
-		return err
+func (p *NATSProvider) UnpublishVolume(ctx context.Context, req UnpublishVolumeRequest) (*UnpublishedVolume, error) {
+	if err := CheckUnpublishVersion(req); err != nil {
+		return nil, err
 	}
 	subject, err := UnpublishSubject(req.NodeID)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	var response UnpublishVolumeResponse
 	if err := p.request(ctx, subject, req, &response); err != nil {
-		return err
+		return nil, err
 	}
-	return responseError(response.SchemaVersion, response.Error)
+	if err := responseError(response.SchemaVersion, response.Error); err != nil {
+		return nil, err
+	}
+	return &UnpublishedVolume{TornDown: response.TornDown}, nil
 }
 
 // HoldsUnsealedWrites returns the subset of volumeIDs for which nodeID's

spinifex/providers/ebs/nats_test.go

@@ -250,7 +250,7 @@ func TestNATSProviderChecksVersionBeforeTransport(t *testing.T) {
 		require.ErrorIs(t, err, ErrUnsupportedVersion)
 	})
 	t.Run("UnpublishVolume", func(t *testing.T) {
-		err := provider.UnpublishVolume(ctx, UnpublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"})
+		_, err := provider.UnpublishVolume(ctx, UnpublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1"})
 		require.ErrorIs(t, err, ErrUnsupportedVersion)
 	})
 }
@@ -563,7 +563,7 @@ func TestNATSProviderPropagatesProviderErrors(t *testing.T) {
 		respondWith(t, conn, subject, UnpublishVolumeResponse{
 			Versioned: NewVersioned(), Error: &ProviderError{Code: ErrorCodeNotFound, Message: "no such mount"},
 		})
-		err = NewNATSProvider(conn, time.Second).UnpublishVolume(t.Context(), UnpublishVolumeRequest{
+		_, err = NewNATSProvider(conn, time.Second).UnpublishVolume(t.Context(), UnpublishVolumeRequest{
 			Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-err",
 		})
 		require.ErrorIs(t, err, ErrNotFound)
@@ -601,7 +601,7 @@ func TestNATSProviderValidatesSubjectTokensBeforeTransport(t *testing.T) {
 		require.ErrorIs(t, err, ErrInvalidArgument)
 	})
 	t.Run("UnpublishVolume", func(t *testing.T) {
-		err := provider.UnpublishVolume(ctx, UnpublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node.*"})
+		_, err := provider.UnpublishVolume(ctx, UnpublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node.*"})
 		require.ErrorIs(t, err, ErrInvalidArgument)
 	})
 }
@@ -712,6 +712,14 @@ func TestTimeoutForSizesEachSubjectToItsWork(t *testing.T) {
 	require.NoError(t, err)
 	ownerDescribe, err := GetVolumeOwnerSubject("vol-1")
 	require.NoError(t, err)
+	unpublish, err := UnpublishSubject("node-1")
+	require.NoError(t, err)
+	publish, err := PublishSubject("node-1")
+	require.NoError(t, err)
+	unsealed, err := UnsealedWritesSubject("node-1")
+	require.NoError(t, err)
+	nodeCapabilities, err := NodeCapabilitiesSubject("node-1")
+	require.NoError(t, err)
 
 	// A delete costs one round trip per object, so its duration is a function
 	// of the volume. On prod a 100 GiB volume's delete took 52 s against a 30 s
@@ -721,6 +729,8 @@ func TestTimeoutForSizesEachSubjectToItsWork(t *testing.T) {
 		CreateVolumeSubject, ExpandVolumeSubject, CopySnapshotSubject,
 		DeleteVolumeSubject, DeleteSnapshotSubject,
 		snapshotCreate, ownerCopy, ownerExpand, ownerCreate,
+		// Unpublish seals the volume's unsealed writes back to the backend.
+		unpublish,
 	}
 	for _, subject := range slow {
 		assert.Equal(t, dataMovementTimeout, p.timeoutFor(subject),
@@ -730,6 +740,7 @@ func TestTimeoutForSizesEachSubjectToItsWork(t *testing.T) {
 	fast := []string{
 		CapabilitiesSubject, GetVolumeSubject, ListVolumesSubject,
 		ListSnapshotsSubject, ownerDescribe,
+		publish, unsealed, nodeCapabilities,
 	}
 	for _, subject := range fast {
 		assert.Equal(t, callerTimeout, p.timeoutFor(subject),

spinifex/providers/ebs/natsserve/classify_test.go

@@ -23,6 +23,7 @@ func TestClassifyError(t *testing.T) {
 		{ebsprovider.ErrVolumeInUse, ebsprovider.ErrorCodeVolumeInUse},
 		{ebsprovider.ErrUnsupportedCapability, ebsprovider.ErrorCodeUnsupportedCap},
 		{ebsprovider.ErrUnavailable, ebsprovider.ErrorCodeUnavailable},
+		{ebsprovider.ErrVolumeHeldElsewhere, ebsprovider.ErrorCodeVolumeHeldElsewhere},
 	}
 
 	for _, tt := range tests {

spinifex/providers/ebs/natsserve/handlers_test.go

@@ -96,8 +96,12 @@ func (p *failingProvider) PublishVolume(context.Context, ebsprovider.PublishVolu
 	return nil, p.err
 }
 
-func (p *failingProvider) UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) error {
-	return p.err
+func (p *failingProvider) UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
+	return nil, p.err
+}
+
+func (p *failingProvider) HoldsUnsealedWrites(context.Context, string, []string) ([]string, error) {
+	return nil, p.err
 }
 
 // wireVerb names one subject Serve subscribes, a valid request body for it,
@@ -120,6 +124,10 @@ func wireVerbs(t *testing.T) []wireVerb {
 	require.NoError(t, err)
 	unmountSubject, err := ebsprovider.UnpublishSubject("node-1")
 	require.NoError(t, err)
+	unsealedSubject, err := ebsprovider.UnsealedWritesSubject("node-1")
+	require.NoError(t, err)
+	nodeCapabilitiesSubject, err := ebsprovider.NodeCapabilitiesSubject("node-1")
+	require.NoError(t, err)
 
 	return []wireVerb{
 		{name: "capabilities", subject: ebsprovider.CapabilitiesSubject,
@@ -146,9 +154,122 @@ func wireVerbs(t *testing.T) []wireVerb {
 			request: ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", NodeID: "node-1"}},
 		{name: "unmount", subject: unmountSubject,
 			request: ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", NodeID: "node-1"}},
+		{name: "unsealed", subject: unsealedSubject,
+			request: ebsprovider.HoldsUnsealedWritesRequest{Versioned: ebsprovider.NewVersioned(), NodeID: "node-1", VolumeIDs: []string{"vol-1"}}},
+		{name: "node.capabilities", subject: nodeCapabilitiesSubject,
+			request: ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}},
 	}
 }
 
+// withVersion re-encodes a wireVerb request at version, so one table drives
+// every served version without a copy of each request per version.
+func withVersion(t *testing.T, request any, version uint16) []byte {
+	t.Helper()
+	payload, err := json.Marshal(request)
+	require.NoError(t, err)
+	var body map[string]any
+	require.NoError(t, json.Unmarshal(payload, &body))
+	body["schema_version"] = version
+	payload, err = json.Marshal(body)
+	require.NoError(t, err)
+	return payload
+}
+
+// TestHandlersReplyInRequestVersion is what lets a version 3 server serve a
+// v1.23.0 caller: every reply, success or provider failure, carries the
+// version the request was sent at, never the server's newest.
+func TestHandlersReplyInRequestVersion(t *testing.T) {
+	failing := serveProvider(t, &failingProvider{err: fmt.Errorf("vol-1: %w", ebsprovider.ErrUnavailable)}, natsserve.Options{NoQueueGroup: true})
+	for _, version := range ebsprovider.SupportedSchemaVersions() {
+		for _, verb := range wireVerbs(t) {
+			// Each success case gets its own seeded provider, so none relies on
+			// an earlier case having created or deleted something.
+			t.Run(fmt.Sprintf("success/v%d/%s", version, verb.name), func(t *testing.T) {
+				conn := serveProvider(t, seededProvider(t), natsserve.Options{NoQueueGroup: true})
+				for _, envelope := range requestAt(t, conn, verb, version) {
+					require.Nil(t, envelope.Error, "the case must succeed for it to check a success reply")
+					assert.Equal(t, version, envelope.SchemaVersion)
+				}
+			})
+			t.Run(fmt.Sprintf("failure/v%d/%s", version, verb.name), func(t *testing.T) {
+				envelopes := requestAt(t, failing, verb, version)
+				// A snapshot create is accepted first; the failure is in its completion.
+				require.NotNil(t, envelopes[len(envelopes)-1].Error, "the case must fail for it to check a failure reply")
+				for _, envelope := range envelopes {
+					assert.Equal(t, version, envelope.SchemaVersion)
+				}
+			})
+		}
+	}
+}
+
+// seededProvider holds what every wireVerbs request needs to succeed: vol-1
+// and snap-1 taken from it, with enumeration advertised so listing succeeds.
+func seededProvider(t *testing.T) ebsprovider.EBSProvider {
+	t.Helper()
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeEnumeration: true, SnapshotEnumeration: true})
+	ctx := context.Background()
+	_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+	require.NoError(t, err)
+	_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: "snap-1", VolumeID: "vol-1"})
+	require.NoError(t, err)
+	return provider
+}
+
+// requestAt sends verb at version and returns the reply envelope, followed by
+// the completion's for a verb that publishes one.
+func requestAt(t *testing.T, conn *nats.Conn, verb wireVerb, version uint16) []wireEnvelope {
+	t.Helper()
+	var completionSub *nats.Subscription
+	if verb.completionSubject != "" {
+		var err error
+		completionSub, err = conn.SubscribeSync(verb.completionSubject)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = completionSub.Unsubscribe() })
+		require.NoError(t, conn.Flush())
+	}
+
+	msg, err := conn.Request(verb.subject, withVersion(t, verb.request, version), requestTimeout)
+	require.NoError(t, err)
+	envelopes := []wireEnvelope{decodeEnvelope(t, msg.Data)}
+	if completionSub != nil {
+		msg, err = completionSub.NextMsg(requestTimeout)
+		require.NoError(t, err)
+		envelopes = append(envelopes, decodeEnvelope(t, msg.Data))
+	}
+	return envelopes
+}
+
+// TestUnpublishNoSealOnTheWire pins the server half of no_seal: refused below
+// version 3 without reaching the provider, and reported as torn down at 3.
+func TestUnpublishNoSealOnTheWire(t *testing.T) {
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
+	conn := serveProvider(t, provider, natsserve.Options{NoQueueGroup: true})
+	ctx := context.Background()
+	_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", NodeID: "node-1"})
+	require.NoError(t, err)
+	subject, err := ebsprovider.UnpublishSubject("node-1")
+	require.NoError(t, err)
+	request := ebsprovider.UnpublishVolumeRequest{VolumeID: "vol-1", NodeID: "node-1", NoSeal: true}
+
+	msg, err := conn.Request(subject, withVersion(t, request, ebsprovider.DescriptorSchemaVersion), requestTimeout)
+	require.NoError(t, err)
+	envelope := decodeEnvelope(t, msg.Data)
+	require.NotNil(t, envelope.Error)
+	assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, envelope.Error.Code)
+	err = provider.DeleteVolume(ctx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1"})
+	require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse, "the refused unpublish must not have reached the provider")
+
+	msg, err = conn.Request(subject, withVersion(t, request, ebsprovider.NoSealSchemaVersion), requestTimeout)
+	require.NoError(t, err)
+	var resp ebsprovider.UnpublishVolumeResponse
+	require.NoError(t, json.Unmarshal(msg.Data, &resp))
+	require.Nil(t, resp.Error)
+	assert.True(t, resp.TornDown)
+}
+
 // TestHandlersRejectMalformedRequests pins the caller-fault arm of every
 // subject: a body that is not JSON must come back as a versioned
 // invalid_argument reply, not as a dropped message the caller waits out.
@@ -184,7 +305,8 @@ func TestHandlersRejectVersionSkew(t *testing.T) {
 			assert.Equal(t, ebsprovider.LegacySchemaVersion, envelope.SchemaVersion, "the refusal itself must be versioned")
 			require.NotNil(t, envelope.Error)
 			assert.Equal(t, ebsprovider.ErrorCodeUnsupportedVersion, envelope.Error.Code)
-			assert.Equal(t, fmt.Sprintf("unsupported schema version 99; supported versions are %d and %d", ebsprovider.LegacySchemaVersion, ebsprovider.SchemaVersion), envelope.Error.Message)
+			assert.Contains(t, envelope.Error.Message, "99", "the refusal must name the version it refused")
+			assert.Contains(t, envelope.Error.Message, ebsprovider.SupportedVersionsText(), "the refusal must name the versions served")
 		})
 	}
 }
@@ -282,9 +404,10 @@ func TestServedProviderRoundTrip(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, ebsprovider.VolumeStateInUse, inUse.State)
 
-	require.NoError(t, client.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
+	_, err = client.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-roundtrip", NodeID: "node-1",
-	}))
+	})
+	require.NoError(t, err)
 	available, err := client.GetVolume(ctx, ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-roundtrip"})
 	require.NoError(t, err)
 	assert.Equal(t, ebsprovider.VolumeStateAvailable, available.State)
@@ -614,6 +737,22 @@ func TestServeNodeIDScopesMountSubjects(t *testing.T) {
 	require.ErrorIs(t, err, nats.ErrNoResponders, "another node's mount subject must go unanswered")
 }
 
+// TestGetNodeCapabilitiesIsAnsweredOnlyByThatNode is why the per-node subject
+// exists: the shared one is answered by whichever node's provider is fastest,
+// which says nothing about the provider on the node a caller will mount on.
+func TestGetNodeCapabilitiesIsAnsweredOnlyByThatNode(t *testing.T) {
+	conn := serveProvider(t, ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}), natsserve.Options{NodeID: "node-1"})
+	client := ebsprovider.NewNATSProvider(conn, requestTimeout)
+	request := ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}
+
+	resp, err := client.GetNodeCapabilities(t.Context(), "node-1", request)
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.SupportedSchemaVersions(), resp.ContractVersions)
+
+	_, err = client.GetNodeCapabilities(t.Context(), "node-2", request)
+	require.ErrorIs(t, err, nats.ErrNoResponders, "another node's capabilities subject must go unanswered")
+}
+
 // TestServeRejectsUnroutableNodeID covers the subject-build failure path: an
 // unsubscribable node ID must fail Serve outright rather than leave a server
 // running with the shared subjects subscribed and no mount subject at all.
@@ -622,7 +761,7 @@ func TestServeRejectsUnroutableNodeID(t *testing.T) {
 
 	stop, err := natsserve.Serve(t.Context(), conn, ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}), natsserve.Options{NodeID: "node.*"})
 	require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
-	assert.Contains(t, err.Error(), "build publish subject")
+	assert.Contains(t, err.Error(), "build per-node subject for "+ebsprovider.PublishSubjectWildcard)
 	assert.Nil(t, stop)
 
 	_, err = conn.Request(ebsprovider.CapabilitiesSubject, []byte(fmt.Sprintf(`{"schema_version":%d}`, ebsprovider.SchemaVersion)), 200*time.Millisecond)

spinifex/providers/ebs/natsserve/server.go

@@ -36,9 +36,9 @@ type Options struct {
 	// across workers does not apply.
 	NoQueueGroup bool
 
-	// NodeID scopes PublishVolume/UnpublishVolume to this node's own
-	// subjects (ebsprovider.PublishSubject / UnpublishSubject), matching a
-	// production daemon that can only mount volumes on its own node. Left empty (the default), Serve instead subscribes the wildcard "ebs.provider.v1.*.mount"/"unmount" subjects, answering requests for any node.
+	// NodeID scopes the per-node subjects (publish, unpublish, unsealed writes
+	// and per-node capabilities) to this node, as a production daemon serves
+	// them. Left empty, Serve subscribes their wildcards and answers for any node.
 	NodeID string
 }
 
@@ -127,31 +127,27 @@ func Serve(ctx context.Context, nc *nats.Conn, provider ebsprovider.EBSProvider,
 		}
 	}
 
-	if opts.NodeID == "" {
-		if err := subscribePlain("ebs.provider.v1.*.mount", handlePublishVolume(provider)); err != nil {
-			cleanup()
-			return nil, err
+	nodeSteps := []struct {
+		wildcard string
+		build    func(string) (string, error)
+		handler  msgHandler
+	}{
+		{ebsprovider.PublishSubjectWildcard, ebsprovider.PublishSubject, handlePublishVolume(provider)},
+		{ebsprovider.UnpublishSubjectWildcard, ebsprovider.UnpublishSubject, handleUnpublishVolume(provider)},
+		{ebsprovider.UnsealedWritesSubjectWildcard, ebsprovider.UnsealedWritesSubject, handleHoldsUnsealedWrites(provider)},
+		{ebsprovider.NodeCapabilitiesSubjectWildcard, ebsprovider.NodeCapabilitiesSubject, handleCapabilities(provider)},
+	}
+	for _, step := range nodeSteps {
+		subject := step.wildcard
+		if opts.NodeID != "" {
+			built, err := step.build(opts.NodeID)
+			if err != nil {
+				cleanup()
+				return nil, fmt.Errorf("build per-node subject for %s: %w", step.wildcard, err)
+			}
+			subject = built
 		}
-		if err := subscribePlain("ebs.provider.v1.*.unmount", handleUnpublishVolume(provider)); err != nil {
-			cleanup()
-			return nil, err
-		}
-	} else {
-		publishSubject, err := ebsprovider.PublishSubject(opts.NodeID)
-		if err != nil {
-			cleanup()
-			return nil, fmt.Errorf("build publish subject: %w", err)
-		}
-		if err := subscribePlain(publishSubject, handlePublishVolume(provider)); err != nil {
-			cleanup()
-			return nil, err
-		}
-		unpublishSubject, err := ebsprovider.UnpublishSubject(opts.NodeID)
-		if err != nil {
-			cleanup()
-			return nil, fmt.Errorf("build unpublish subject: %w", err)
-		}
-		if err := subscribePlain(unpublishSubject, handleUnpublishVolume(provider)); err != nil {
+		if err := subscribePlain(subject, step.handler); err != nil {
 			cleanup()
 			return nil, err
 		}
@@ -188,7 +184,7 @@ func badRequestError(err error) *ebsprovider.ProviderError {
 }
 
 func versionError(got uint16) *ebsprovider.ProviderError {
-	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: fmt.Sprintf("unsupported schema version %d; supported versions are %d and %d", got, ebsprovider.LegacySchemaVersion, ebsprovider.SchemaVersion)}
+	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: fmt.Sprintf("unsupported schema version %d; %s", got, ebsprovider.SupportedVersionsText())}
 }
 
 func invalidArgumentError(format string, args ...any) *ebsprovider.ProviderError {
@@ -202,6 +198,7 @@ func internalError(format string, args ...any) *ebsprovider.ProviderError {
 // classifyError maps a provider error to the wire ProviderError shape,
 // using only the sentinel errors and ErrorCode values errors.go exports:
 // it has no forward (error -> ProviderError) mapper of its own to call.
+// The provider picks the sentinel for the request's version (HeldElsewhereError).
 func classifyError(err error) *ebsprovider.ProviderError {
 	switch {
 	case errors.Is(err, ebsprovider.ErrAlreadyExists):
@@ -212,6 +209,8 @@ func classifyError(err error) *ebsprovider.ProviderError {
 		return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeNotFound, Message: err.Error()}
 	case errors.Is(err, ebsprovider.ErrUnsupportedVersion):
 		return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: err.Error()}
+	case errors.Is(err, ebsprovider.ErrVolumeHeldElsewhere):
+		return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeHeldElsewhere, Message: err.Error()}
 	case errors.Is(err, ebsprovider.ErrVolumeInUse):
 		return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: err.Error()}
 	case errors.Is(err, ebsprovider.ErrUnsupportedCapability):
@@ -226,178 +225,202 @@ func classifyError(err error) *ebsprovider.ProviderError {
 func handleCapabilities(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.GetCapabilitiesRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		resp, err := provider.GetCapabilities(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, *resp)
+		reply := *resp
+		reply.Versioned = v
+		respond(ctx, msg, reply)
 	}
 }
 
 func handleCreateVolume(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.CreateVolumeRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		vol, err := provider.CreateVolume(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: vol})
+		respond(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Volume: vol})
 	}
 }
 
 func handleGetVolume(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.GetVolumeRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		vol, err := provider.GetVolume(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: vol})
+		respond(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Volume: vol})
 	}
 }
 
 func handleListVolumes(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.ListVolumesRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		resp, err := provider.ListVolumes(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, *resp)
+		reply := *resp
+		reply.Versioned = v
+		respond(ctx, msg, reply)
 	}
 }
 
 func handleListSnapshots(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.ListSnapshotsRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		resp, err := provider.ListSnapshots(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, *resp)
+		reply := *resp
+		reply.Versioned = v
+		respond(ctx, msg, reply)
 	}
 }
 
 func handleExpandVolume(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.ExpandVolumeRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		vol, err := provider.ExpandVolume(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: vol})
+		respond(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Volume: vol})
 	}
 }
 
 func handleDeleteVolume(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.DeleteVolumeRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		if err := provider.DeleteVolume(ctx, req); err != nil {
-			respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned()})
+		respond(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v})
 	}
 }
 
 func handleDeleteSnapshot(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.DeleteSnapshotRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		if err := provider.DeleteSnapshot(ctx, req); err != nil {
-			respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned()})
+		respond(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v})
 	}
 }
 
 func handleCopySnapshot(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.CopySnapshotRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		snap, err := provider.CopySnapshot(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Snapshot: snap})
+		respond(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Snapshot: snap})
 	}
 }
 
@@ -407,12 +430,14 @@ func handleCopySnapshot(provider ebsprovider.EBSProvider) msgHandler {
 func handleCreateSnapshot(nc *nats.Conn, provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.CreateSnapshotRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 
@@ -420,23 +445,23 @@ func handleCreateSnapshot(nc *nats.Conn, provider ebsprovider.EBSProvider) msgHa
 		// token (SnapshotCreateSubjectPrefix + volumeID); a mismatch with
 		// the request body means the two disagree about which volume this create is for.
 		if subjectVolumeID, ok := strings.CutPrefix(msg.Subject, ebsprovider.SnapshotCreateSubjectPrefix); ok && subjectVolumeID != req.VolumeID {
-			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("volume id %q in subject does not match request volume id %q", subjectVolumeID, req.VolumeID)})
+			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: invalidArgumentError("volume id %q in subject does not match request volume id %q", subjectVolumeID, req.VolumeID)})
 			return
 		}
 
 		completionSubject, err := ebsprovider.SnapshotCompletionSubject(req.SnapshotID)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
 		operationID, err := newOperationID()
 		if err != nil {
-			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("generate operation id: %v", err)})
+			respond(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: internalError("generate operation id: %v", err)})
 			return
 		}
 
 		respond(ctx, msg, ebsprovider.CreateSnapshotResponse{
-			Versioned:         ebsprovider.NewVersioned(),
+			Versioned:         v,
 			OperationID:       operationID,
 			CompletionSubject: completionSubject,
 			Snapshot: &ebsprovider.Snapshot{
@@ -452,7 +477,7 @@ func handleCreateSnapshot(nc *nats.Conn, provider ebsprovider.EBSProvider) msgHa
 
 func completeCreateSnapshot(ctx context.Context, nc *nats.Conn, provider ebsprovider.EBSProvider, req ebsprovider.CreateSnapshotRequest, operationID, completionSubject string) {
 	completion := ebsprovider.CreateSnapshotResponse{
-		Versioned:         ebsprovider.NewVersioned(),
+		Versioned:         ebsprovider.ReplyVersioned(req.SchemaVersion),
 		OperationID:       operationID,
 		CompletionSubject: completionSubject,
 	}
@@ -488,27 +513,29 @@ func newOperationID() (string, error) {
 func handlePublishVolume(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.PublishVolumeRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
 		pub, err := provider.PublishVolume(ctx, req)
 		if err != nil {
-			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
 		if pub == nil {
-			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("provider returned no published volume for %s", req.VolumeID)})
+			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: internalError("provider returned no published volume for %s", req.VolumeID)})
 			return
 		}
 		// An unattachable export is the provider's fault, so it is reported as
 		// internal rather than relayed for the client to reject.
 		if err := pub.Export.Validate(); err != nil {
-			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("provider returned an invalid export for %s: %v", req.VolumeID, err)})
+			respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: internalError("provider returned an invalid export for %s: %v", req.VolumeID, err)})
 			return
 		}
 		if req.SchemaVersion == ebsprovider.LegacySchemaVersion {
@@ -517,36 +544,65 @@ func handlePublishVolume(provider ebsprovider.EBSProvider) msgHandler {
 				// A v1 caller cannot safely consume a block export. Do not emit an
 				// empty legacy URI and leave the old client to misattach it.
 				respond(ctx, msg, ebsprovider.PublishVolumeResponseV1{
-					Versioned: ebsprovider.NewVersioned(),
+					Versioned: v,
 					Error:     &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: err.Error()},
 				})
 				return
 			}
 			respond(ctx, msg, ebsprovider.PublishVolumeResponseV1{
-				Versioned: ebsprovider.NewVersioned(),
+				Versioned: v,
 				Published: legacy,
 			})
 			return
 		}
-		respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewLatestVersioned(), Published: pub})
+		respond(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Published: pub})
 	}
 }
 
 func handleUnpublishVolume(provider ebsprovider.EBSProvider) msgHandler {
 	return func(ctx context.Context, msg *nats.Msg) {
 		var req ebsprovider.UnpublishVolumeRequest
-		if err := json.Unmarshal(msg.Data, &req); err != nil {
-			respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: badRequestError(err)})
+			return
+		}
+		if err := ebsprovider.CheckUnpublishVersion(req); err != nil {
+			respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: classifyError(err)})
+			return
+		}
+		result, err := provider.UnpublishVolume(ctx, req)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: classifyError(err)})
+			return
+		}
+		response := ebsprovider.UnpublishVolumeResponse{Versioned: v}
+		if result != nil {
+			response.TornDown = result.TornDown
+		}
+		respond(ctx, msg, response)
+	}
+}
+
+func handleHoldsUnsealedWrites(provider ebsprovider.EBSProvider) msgHandler {
+	return func(ctx context.Context, msg *nats.Msg) {
+		var req ebsprovider.HoldsUnsealedWritesRequest
+		err := json.Unmarshal(msg.Data, &req)
+		v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: badRequestError(err)})
 			return
 		}
 		if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-			respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: versionError(req.SchemaVersion)})
+			respond(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: versionError(req.SchemaVersion)})
 			return
 		}
-		if err := provider.UnpublishVolume(ctx, req); err != nil {
-			respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: classifyError(err)})
+		held, err := provider.HoldsUnsealedWrites(ctx, req.NodeID, req.VolumeIDs)
+		if err != nil {
+			respond(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: classifyError(err)})
 			return
 		}
-		respond(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned()})
+		respond(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Held: held})
 	}
 }

spinifex/providers/ebs/provider.go

@@ -19,10 +19,20 @@ const (
 	// evidence that the peer supports a newer representation.
 	LegacySchemaVersion uint16 = 1
 
+	// DescriptorSchemaVersion is the first version whose PublishVolume reply
+	// carries a VolumeExport descriptor, and so the first that can describe a
+	// block export.
+	DescriptorSchemaVersion uint16 = 2
+
+	// NoSealSchemaVersion is the first version that carries
+	// UnpublishVolumeRequest.NoSeal. A provider that does not serve it refuses
+	// the request as unsupported rather than dropping the flag and sealing.
+	NoSealSchemaVersion uint16 = 3
+
 	// SchemaVersion is the newest wire-contract version understood by this
 	// build. Every request and response carries a version so an unsupported
 	// representation fails at the boundary rather than silently decoding it.
-	SchemaVersion uint16 = 2
+	SchemaVersion uint16 = NoSealSchemaVersion
 )
 
 // Versioned is embedded in every request and response sent over NATS.
@@ -41,14 +51,39 @@ func NewVersioned() Versioned { return Versioned{SchemaVersion: LegacySchemaVers
 func NewLatestVersioned() Versioned { return Versioned{SchemaVersion: SchemaVersion} }
 
 // SupportsSchemaVersion reports whether this build can safely decode version.
+// Every version from LegacySchemaVersion to SchemaVersion is served.
 func SupportsSchemaVersion(version uint16) bool {
-	return version == LegacySchemaVersion || version == SchemaVersion
+	return version >= LegacySchemaVersion && version <= SchemaVersion
+}
+
+// SupportedSchemaVersions lists, in ascending order, every contract version
+// this build serves. GetCapabilitiesResponse.ContractVersions reports it.
+func SupportedSchemaVersions() []uint16 {
+	versions := make([]uint16, 0, SchemaVersion-LegacySchemaVersion+1)
+	for v := LegacySchemaVersion; v <= SchemaVersion; v++ {
+		versions = append(versions, v)
+	}
+	return versions
+}
+
+// ReplyVersioned stamps a reply with its request's version, so a body is never
+// labelled newer than its shape. A version this build cannot serve is answered
+// at LegacySchemaVersion, the one every peer decodes.
+func ReplyVersioned(requested uint16) Versioned {
+	if !SupportsSchemaVersion(requested) {
+		return NewVersioned()
+	}
+	return Versioned{SchemaVersion: requested}
 }
 
 // EBSProvider is the block-storage controller contract consumed by Spinifex.
 // Its operation and idempotency shape follows the CSI controller service, but
 // the transport remains NATS request/reply rather than CSI's gRPC transport.
 type EBSProvider interface {
+	// Required of every provider: a caller counts no answer as held, which
+	// would keep an instance from ever starting on another node.
+	UnsealedWritesChecker
+
 	GetCapabilities(context.Context, GetCapabilitiesRequest) (*GetCapabilitiesResponse, error)
 	CreateVolume(context.Context, CreateVolumeRequest) (*Volume, error)
 	GetVolume(context.Context, GetVolumeRequest) (*Volume, error)
@@ -60,7 +95,7 @@ type EBSProvider interface {
 	CopySnapshot(context.Context, CopySnapshotRequest) (*Snapshot, error)
 	ListSnapshots(context.Context, ListSnapshotsRequest) (*ListSnapshotsResponse, error)
 	PublishVolume(context.Context, PublishVolumeRequest) (*PublishedVolume, error)
-	UnpublishVolume(context.Context, UnpublishVolumeRequest) error
+	UnpublishVolume(context.Context, UnpublishVolumeRequest) (*UnpublishedVolume, error)
 }
 
 // Provider is retained as a concise alias for implementation code. New
@@ -164,8 +199,14 @@ type GetCapabilitiesRequest struct{ Versioned }
 type GetCapabilitiesResponse struct {
 	Versioned
 
-	Capabilities Capabilities   `json:"capabilities"`
-	Error        *ProviderError `json:"error,omitempty"`
+	Capabilities Capabilities `json:"capabilities"`
+
+	// ContractVersions lists every schema version the answering provider
+	// serves, ascending. A provider that predates the field leaves it empty,
+	// which a caller must read as no evidence, not as version 1 only.
+	ContractVersions []uint16 `json:"contract_versions,omitempty"`
+
+	Error *ProviderError `json:"error,omitempty"`
 }
 
 // CapacityRange mirrors CSI's capacity-range semantics. RequiredBytes is the
@@ -563,7 +604,7 @@ func (p *PublishedVolume) AsV1() (*PublishedVolumeV1, error) {
 		return nil, err
 	}
 	if p.Export.Kind != ExportKindNBD {
-		return nil, fmt.Errorf("%s export requires schema version %d", p.Export.Kind, SchemaVersion)
+		return nil, fmt.Errorf("%s export requires schema version %d", p.Export.Kind, DescriptorSchemaVersion)
 	}
 	return &PublishedVolumeV1{
 		VolumeID: p.VolumeID,
@@ -617,6 +658,31 @@ type UnpublishVolumeRequest struct {
 	VolumeID string `json:"volume_id"`
 	Handle   string `json:"handle,omitempty"`
 	NodeID   string `json:"node_id"`
+
+	// NoSeal drops this node's copy unsealed, because a newer owner's supersedes
+	// it; unlike AWS DetachVolume Force, which leaves a copy that must seal.
+	// Carried only at NoSealSchemaVersion or later (CheckUnpublishVersion).
+	NoSeal bool `json:"no_seal,omitempty"`
+}
+
+// CheckUnpublishVersion refuses a request this build cannot serve, and a
+// NoSeal sent below NoSealSchemaVersion: a provider at that version would
+// drop the flag and seal, so no provider may honour it there either.
+func CheckUnpublishVersion(req UnpublishVolumeRequest) error {
+	if err := checkVersion(req.SchemaVersion); err != nil {
+		return err
+	}
+	if req.NoSeal && req.SchemaVersion < NoSealSchemaVersion {
+		return fmt.Errorf("%w: no_seal requires schema version %d, got %d", ErrInvalidArgument, NoSealSchemaVersion, req.SchemaVersion)
+	}
+	return nil
+}
+
+// UnpublishedVolume is the outcome of an UnpublishVolume that succeeded.
+// TornDown reports whether an export existed and was removed; it is reported
+// only at NoSealSchemaVersion or later, and is false at earlier versions.
+type UnpublishedVolume struct {
+	TornDown bool
 }
 
 // UnpublishVolumeResponse is the reply to UnpublishVolumeRequest. A nil Error
@@ -624,13 +690,14 @@ type UnpublishVolumeRequest struct {
 type UnpublishVolumeResponse struct {
 	Versioned
 
-	Error *ProviderError `json:"error,omitempty"`
+	// TornDown mirrors UnpublishedVolume.TornDown.
+	TornDown bool           `json:"torn_down,omitempty"`
+	Error    *ProviderError `json:"error,omitempty"`
 }
 
 // UnsealedWritesChecker answers, for one node, which of a set of volumes it
-// still holds writes for that the backend may not have. It is separate from
-// EBSProvider because only the instance start path asks, and only of the node
-// that last ran the instance.
+// still holds writes for that the backend may not have. A write-through
+// provider answers with none. Consumers that only ask this depend on it alone.
 type UnsealedWritesChecker interface {
 	HoldsUnsealedWrites(ctx context.Context, nodeID string, volumeIDs []string) ([]string, error)
 }
@@ -652,3 +719,25 @@ type HoldsUnsealedWritesResponse struct {
 	Held  []string       `json:"held,omitempty"`
 	Error *ProviderError `json:"error,omitempty"`
 }
+
+// FenceNotice announces that a provider revoked a live export on NodeID
+// because another node now holds the volume, so that node's daemon can stop
+// the guest using it. Sent over FenceNoticeSubject until acknowledged.
+type FenceNotice struct {
+	Versioned
+
+	VolumeID string `json:"volume_id"`
+	NodeID   string `json:"node_id"`
+	Winner   string `json:"winner,omitempty"`
+	Reason   string `json:"reason,omitempty"`
+}
+
+// FenceNoticeAck is the daemon's answer to a FenceNotice. Any answer ends the
+// sender's retries; InstanceID names the guest stopped, if one was, and Error
+// says why the daemon could not act on the notice.
+type FenceNoticeAck struct {
+	Versioned
+
+	InstanceID string         `json:"instance_id,omitempty"`
+	Error      *ProviderError `json:"error,omitempty"`
+}

spinifex/providers/ebs/telemetry.go

@@ -46,6 +46,8 @@ const (
 	verbVolumeUnpublish = "volume.unpublish"
 	verbSnapshotDelete  = "snapshot.delete"
 	verbSnapshotList    = "snapshot.list"
+	verbUnsealedWrites  = "volume.unsealed"
+	verbFenceNotice     = "volume.fenced"
 )
 
 // subjectVerbs maps every fixed subject to its verb. The dynamic subject
@@ -107,10 +109,16 @@ func parseSubject(subject string) subjectTarget {
 		return subjectTarget{verb: VerbUnknown}
 	}
 	switch suffix {
-	case "mount":
+	case publishVerb:
 		return subjectTarget{verb: verbVolumePublish, nodeID: nodeID}
-	case "unmount":
+	case unpublishVerb:
 		return subjectTarget{verb: verbVolumeUnpublish, nodeID: nodeID}
+	case unsealedVerb:
+		return subjectTarget{verb: verbUnsealedWrites, nodeID: nodeID}
+	case nodeCapabilitiesVerb:
+		return subjectTarget{verb: verbCapabilities, nodeID: nodeID}
+	case fenceNoticeVerb:
+		return subjectTarget{verb: verbFenceNotice, nodeID: nodeID}
 	default:
 		return subjectTarget{verb: VerbUnknown}
 	}

spinifex/providers/ebs/telemetry_test.go

@@ -30,6 +30,12 @@ func TestSubjectVerb_StripsIDsFromEveryDynamicSubject(t *testing.T) {
 	require.NoError(t, err)
 	unpublish, err := UnpublishSubject(nodeID)
 	require.NoError(t, err)
+	unsealed, err := UnsealedWritesSubject(nodeID)
+	require.NoError(t, err)
+	nodeCapabilities, err := NodeCapabilitiesSubject(nodeID)
+	require.NoError(t, err)
+	fenced, err := FenceNoticeSubject(nodeID)
+	require.NoError(t, err)
 
 	cases := map[string]string{
 		CapabilitiesSubject:   verbCapabilities,
@@ -47,6 +53,9 @@ func TestSubjectVerb_StripsIDsFromEveryDynamicSubject(t *testing.T) {
 		snapshotCreate:        verbSnapshotCreate,
 		publish:               verbVolumePublish,
 		unpublish:             verbVolumeUnpublish,
+		unsealed:              verbUnsealedWrites,
+		nodeCapabilities:      verbCapabilities,
+		fenced:                verbFenceNotice,
 	}
 
 	for subject, want := range cases {

spinifex/providers/ebs/viperblock/conformance_test.go

@@ -15,6 +15,7 @@ import (
 	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs/conformance"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	testpredastore "github.com/mulgadc/spinifex/tests/fixtures/predastore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/require"
@@ -61,7 +62,8 @@ func requireExportTools(t *testing.T) string {
 // would go to a subject nobody serves and time out instead of being refused.
 // Both share the NATS the volume leases live in and the bucket the volumes
 // live in, so a second opener is refused by the lease, not by the harness.
-func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider, string, string) {
+// The last return kills a published export's nbdkit behind the handlers' back.
+func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider, string, string, func(*testing.T, *ebsprovider.PublishedVolume)) {
 	t.Helper()
 	pluginPath := requireExportTools(t)
 
@@ -98,10 +100,21 @@ func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider
 	_, err = js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
 	require.NoError(t, err)
 
+	killExport := func(t *testing.T, published *ebsprovider.PublishedVolume) {
+		t.Helper()
+		for _, node := range []*Config{first, second} {
+			if mv, ok := findMountedVolume(node, published.VolumeID); ok && node.NodeName == published.NodeID {
+				require.NoError(t, hostprocess.ForceKillProcess(mv.PID, 5*time.Second))
+				return
+			}
+		}
+		t.Fatalf("no mounted entry for %s on %s", published.VolumeID, published.NodeID)
+	}
+
 	return func(t *testing.T) ebsprovider.EBSProvider {
 		t.Helper()
 		return ebsprovider.NewNATSProvider(nc, 60*time.Second)
-	}, first.NodeName, second.NodeName
+	}, first.NodeName, second.NodeName, killExport
 }
 
 // runPrefix names one run's volumes so it cannot meet an earlier run's
@@ -113,11 +126,12 @@ func runPrefix() string {
 // TestViperblockdConformance judges the provider handlers by the same suite
 // MemoryProvider and qemunbdd answer to.
 func TestViperblockdConformance(t *testing.T) {
-	newProvider, nodeID, otherNodeID := inProcessProvider(t)
+	newProvider, nodeID, otherNodeID, killExport := inProcessProvider(t)
 	conformance.RunSuiteWithConfig(t, newProvider, conformance.SuiteConfig{
 		NamePrefix:  runPrefix(),
 		NodeID:      nodeID,
 		OtherNodeID: otherNodeID,
+		KillExport:  killExport,
 	})
 }
 
@@ -125,7 +139,7 @@ func TestViperblockdConformance(t *testing.T) {
 // cannot: whether the nbdkit export viperblockd publishes is usable by libnbd,
 // which knows nothing about this codebase.
 func TestViperblockdNBDClient(t *testing.T) {
-	newProvider, nodeID, _ := inProcessProvider(t)
+	newProvider, nodeID, _, _ := inProcessProvider(t)
 	conformance.RunNBDClientSuiteWithConfig(t, newProvider, conformance.NBDClientConfig{
 		NodeID:       nodeID,
 		VolumePrefix: "vol-" + runPrefix() + "nbd",

spinifex/providers/ebs/viperblock/provider_contract_test.go

@@ -0,0 +1,339 @@
+package viperblock
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"maps"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/testkit/deadpid"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
+	"github.com/mulgadc/viperblock/viperblock"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestProviderHandlers_RepliesInRequestVersion holds every subject to the
+// rule a mixed-version rollout depends on: a caller at version N must get a
+// reply at N, success or error, or it may not be able to parse it.
+func TestProviderHandlers_RepliesInRequestVersion(t *testing.T) {
+	prevProbe := ownerProbeTimeout
+	ownerProbeTimeout = 100 * time.Millisecond
+	t.Cleanup(func() { ownerProbeTimeout = prevProbe })
+
+	_, natsURL := setupEmbeddedNATS(t)
+	cfg := setupTestConfig(t, natsURL)
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	for _, version := range ebsprovider.SupportedSchemaVersions() {
+		for _, tc := range providerSubjectCases() {
+			t.Run(fmt.Sprintf("v%d/%s", version, tc.name), func(t *testing.T) {
+				body := map[string]any{"schema_version": version}
+				maps.Copy(body, tc.body)
+				msg := requestProvider(t, nc, tc.subject, marshalRequest(t, body))
+
+				var resp versionedErrorResponse
+				require.NoError(t, json.Unmarshal(msg.Data, &resp))
+				assert.Equal(t, version, resp.SchemaVersion)
+			})
+		}
+	}
+}
+
+// TestProviderHandlers_UnpublishNoSealNeverSeals is the reason no_seal exists:
+// a node whose copy is behind a newer owner's must drop it, and a seal would
+// write that stale copy over the owner's block map.
+func TestProviderHandlers_UnpublishNoSealNeverSeals(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-unpubnoseal01"
+	sealed := false
+	cfg := fencedConfig(t, natsURL, "test-node", volumeName)
+	withLocalState(t, cfg, volumeName)
+	cfg.sealVolume = func(context.Context, string) error { sealed = true; return nil }
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	subject, err := ebsprovider.UnpublishSubject(cfg.NodeName)
+	require.NoError(t, err)
+	body := marshalRequest(t, map[string]any{
+		"schema_version": ebsprovider.NoSealSchemaVersion,
+		"volume_id":      volumeName,
+		"node_id":        cfg.NodeName,
+		"no_seal":        true,
+	})
+	msg := requestProvider(t, nc, subject, body)
+
+	var resp ebsprovider.UnpublishVolumeResponse
+	require.NoError(t, json.Unmarshal(msg.Data, &resp))
+	require.Nil(t, resp.Error)
+	assert.True(t, resp.TornDown, "the volume was mounted, so the export was torn down")
+	assert.False(t, sealed, "a no_seal unpublish must never seal")
+	_, mounted := findMountedVolume(cfg, volumeName)
+	assert.False(t, mounted, "the entry must go with the export")
+}
+
+// TestProviderHandlers_UnpublishNoSealBelowV3Refused covers the other half: a
+// caller below version 3 cannot ask for no_seal, so the flag is refused rather
+// than ignored, and the export is left alone.
+func TestProviderHandlers_UnpublishNoSealBelowV3Refused(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-unpubnosealv2"
+	sealed := false
+	cfg := fencedConfig(t, natsURL, "test-node", volumeName)
+	cfg.sealVolume = func(context.Context, string) error { sealed = true; return nil }
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	subject, err := ebsprovider.UnpublishSubject(cfg.NodeName)
+	require.NoError(t, err)
+	body := marshalRequest(t, map[string]any{
+		"schema_version": ebsprovider.DescriptorSchemaVersion,
+		"volume_id":      volumeName,
+		"node_id":        cfg.NodeName,
+		"no_seal":        true,
+	})
+	msg := requestProvider(t, nc, subject, body)
+
+	var resp ebsprovider.UnpublishVolumeResponse
+	require.NoError(t, json.Unmarshal(msg.Data, &resp))
+	require.NotNil(t, resp.Error)
+	assert.Equal(t, ebsprovider.ErrorCodeInvalidArgument, resp.Error.Code)
+	assert.False(t, sealed)
+	_, mounted := findMountedVolume(cfg, volumeName)
+	assert.True(t, mounted, "a refused unpublish must leave the export running")
+}
+
+// TestProviderHandlers_PublishVolume_DeadEntryIsNotHandedBack pins the removed
+// early return: an entry whose nbdkit has died must be dropped and remounted,
+// never answered with a URI nothing serves.
+func TestProviderHandlers_PublishVolume_DeadEntryIsNotHandedBack(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+	cfg := setupTestConfig(t, natsURL)
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	const volumeName = "vol-pubdeadentry1"
+	const deadURI = "nbd://127.0.0.1:1"
+	cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{
+		Name: volumeName, VB: createTestVBWithState(t, volumeName), PID: deadpid.PID, NBDURI: deadURI,
+	})
+
+	subject, err := ebsprovider.PublishSubject(cfg.NodeName)
+	require.NoError(t, err)
+	body := marshalRequest(t, map[string]any{
+		"schema_version": ebsprovider.SchemaVersion,
+		"volume_id":      volumeName,
+		"node_id":        cfg.NodeName,
+	})
+	msg := requestProvider(t, nc, subject, body)
+
+	var resp ebsprovider.PublishVolumeResponse
+	require.NoError(t, json.Unmarshal(msg.Data, &resp))
+	assert.True(t, resp.Error != nil || resp.Published != nil, "the reply must be a refusal or an export")
+	assert.NotContains(t, string(msg.Data), deadURI, "the dead entry's URI was handed back")
+	mv, _ := findMountedVolume(cfg, volumeName)
+	assert.NotEqual(t, deadURI, mv.NBDURI, "the dead entry must not survive the republish")
+}
+
+// TestProviderHandlers_PublishVolume_RepublishRepliesInRequestVersion checks a
+// successful publish reply, not only refusals, at every served version: a
+// republish of a live export is the success path that needs no nbdkit.
+func TestProviderHandlers_PublishVolume_RepublishRepliesInRequestVersion(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+	cfg := setupTestConfig(t, natsURL)
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	const volumeName = "vol-republishver1"
+	socket := filepath.Join(t.TempDir(), "nbd.sock")
+	require.NoError(t, os.WriteFile(socket, nil, 0o600))
+	liveURI := nbd.FormatNBDSocketURI(socket)
+	cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{
+		Name: volumeName, VB: createTestVBWithState(t, volumeName), PID: os.Getpid(), Socket: socket, NBDURI: liveURI,
+	})
+
+	subject, err := ebsprovider.PublishSubject(cfg.NodeName)
+	require.NoError(t, err)
+	for _, version := range ebsprovider.SupportedSchemaVersions() {
+		t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) {
+			msg := requestProvider(t, nc, subject, marshalRequest(t, map[string]any{
+				"schema_version": version,
+				"volume_id":      volumeName,
+				"node_id":        cfg.NodeName,
+			}))
+			var resp versionedErrorResponse
+			require.NoError(t, json.Unmarshal(msg.Data, &resp))
+			require.Nil(t, resp.Error, "the republish must succeed for this to check a success reply")
+			assert.Equal(t, version, resp.SchemaVersion)
+			assert.Contains(t, string(msg.Data), socket, "the live export is handed back")
+		})
+	}
+}
+
+// TestProviderHandlers_NodeCapabilitiesListsServedVersions pins the per-node
+// subject to the same contract versions this build serves.
+func TestProviderHandlers_NodeCapabilitiesListsServedVersions(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+	cfg := setupTestConfig(t, natsURL)
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	subject, err := ebsprovider.NodeCapabilitiesSubject(cfg.NodeName)
+	require.NoError(t, err)
+	msg := requestProvider(t, nc, subject, marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion}))
+
+	var resp ebsprovider.GetCapabilitiesResponse
+	require.NoError(t, json.Unmarshal(msg.Data, &resp))
+	require.Nil(t, resp.Error)
+	assert.Equal(t, ebsprovider.SupportedSchemaVersions(), resp.ContractVersions)
+}
+
+// TestProviderHandlers_UnpublishNoSealAboveServedVersionRefused covers a caller
+// newer than this build: a no_seal it cannot vouch for is refused, and the
+// export is neither torn down nor sealed.
+func TestProviderHandlers_UnpublishNoSealAboveServedVersionRefused(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-unpubnosealv4"
+	sealed := false
+	cfg := fencedConfig(t, natsURL, "test-node", volumeName)
+	withLocalState(t, cfg, volumeName)
+	cfg.sealVolume = func(context.Context, string) error { sealed = true; return nil }
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	subject, err := ebsprovider.UnpublishSubject(cfg.NodeName)
+	require.NoError(t, err)
+	msg := requestProvider(t, nc, subject, marshalRequest(t, map[string]any{
+		"schema_version": ebsprovider.SchemaVersion + 1,
+		"volume_id":      volumeName,
+		"node_id":        cfg.NodeName,
+		"no_seal":        true,
+	}))
+
+	var resp ebsprovider.UnpublishVolumeResponse
+	require.NoError(t, json.Unmarshal(msg.Data, &resp))
+	require.NotNil(t, resp.Error)
+	assert.Equal(t, ebsprovider.ErrorCodeUnsupportedVersion, resp.Error.Code)
+	assert.False(t, resp.TornDown)
+	assert.False(t, sealed, "a refused unpublish must not seal")
+	_, mounted := findMountedVolume(cfg, volumeName)
+	assert.True(t, mounted, "a refused unpublish must leave the export running")
+}
+
+// TestPublishError_Codes pins the code each mount failure reaches a caller as.
+// held_elsewhere is gated on the request version because v1.23.0 callers only
+// know volume_in_use, and a mode mismatch is never held_elsewhere.
+func TestPublishError_Codes(t *testing.T) {
+	leaseHeld := fmt.Errorf("acquire: %w", errVolumeLeaseHeld)
+	cases := []struct {
+		name    string
+		version uint16
+		err     error
+		want    ebsprovider.ErrorCode
+	}{
+		{"absent volume", ebsprovider.SchemaVersion, fmt.Errorf("load: %w", viperblock.ErrStateNotFound), ebsprovider.ErrorCodeNotFound},
+		{"lease held at v1", ebsprovider.LegacySchemaVersion, leaseHeld, ebsprovider.ErrorCodeVolumeInUse},
+		{"lease held at v2", ebsprovider.DescriptorSchemaVersion, leaseHeld, ebsprovider.ErrorCodeVolumeInUse},
+		{"lease held at v3", ebsprovider.NoSealSchemaVersion, leaseHeld, ebsprovider.ErrorCodeVolumeHeldElsewhere},
+		{"mode mismatch at v3", ebsprovider.NoSealSchemaVersion, fmt.Errorf("mounted: %w", errReadOnlyMismatch), ebsprovider.ErrorCodeVolumeInUse},
+		{"backing store not ready", ebsprovider.SchemaVersion, fmt.Errorf("load: %w", viperblock.ErrStateBackendUnavailable), ebsprovider.ErrorCodeUnavailable},
+		{"lease store not ready", ebsprovider.SchemaVersion, fmt.Errorf("open: %w", ErrLeaseStoreUnavailable), ebsprovider.ErrorCodeUnavailable},
+		{"anything else", ebsprovider.SchemaVersion, errors.New("nbdkit exited"), ebsprovider.ErrorCodeInternal},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := publishError(tc.version, "vol-publisherror01", tc.err)
+			require.NotNil(t, got)
+			assert.Equal(t, tc.want, got.Code)
+		})
+	}
+	assert.True(t, ebsprovider.ErrorCodeUnavailable.Retryable(), "a backing store that is not ready must read as retry-later")
+}
+
+// TestProviderHandlers_PublishVolume_StoreNotReadyIsUnavailable carries a mount
+// refused because a store is not ready yet to the wire as retry-later, so a
+// caller waits for the store instead of giving up on the volume.
+func TestProviderHandlers_PublishVolume_StoreNotReadyIsUnavailable(t *testing.T) {
+	for _, cause := range []error{ErrLeaseStoreUnavailable, viperblock.ErrStateBackendUnavailable} {
+		t.Run(cause.Error(), func(t *testing.T) {
+			_, natsURL := setupEmbeddedNATS(t)
+			cfg := setupTestConfig(t, natsURL)
+			cfg.constructVB = func(context.Context, string) (*viperblock.VB, int, error) {
+				return nil, 0, fmt.Errorf("open: %w", cause)
+			}
+			nc := startProviderSubjects(t, cfg, natsURL)
+
+			subject, err := ebsprovider.PublishSubject(cfg.NodeName)
+			require.NoError(t, err)
+			msg := requestProvider(t, nc, subject, marshalRequest(t, map[string]any{
+				"schema_version": ebsprovider.SchemaVersion,
+				"volume_id":      "vol-storenotready",
+				"node_id":        cfg.NodeName,
+			}))
+
+			var resp ebsprovider.PublishVolumeResponse
+			require.NoError(t, json.Unmarshal(msg.Data, &resp))
+			require.NotNil(t, resp.Error)
+			assert.Equal(t, ebsprovider.ErrorCodeUnavailable, resp.Error.Code)
+			assert.Nil(t, resp.Published)
+		})
+	}
+}
+
+// TestProviderHandlers_CreateSnapshot_QueueRefusesWhileLeaseHeldElsewhere is
+// what makes a snapshot from a non-owner safe: whichever node answers the queue
+// subject must not snapshot a checkpoint the holder may still be writing.
+func TestProviderHandlers_CreateSnapshot_QueueRefusesWhileLeaseHeldElsewhere(t *testing.T) {
+	const volumeName = "vol-snapqueueheld1"
+	const snapshotID = "snap-queueheld00001"
+
+	// The volume really exists in the backend, so only the lease stands
+	// between this request and a snapshot written beside the holder's writes.
+	key := testMasterKey(t)
+	backend := newRecordingS3(t, "/"+volumeName+"/config.json", seedVolumeConfig(t, volumeName, key))
+	_, natsURL := setupEmbeddedNATS(t)
+	cfg := setupTestConfig(t, natsURL)
+	cfg.S3Host = backend.URL
+	cfg.masterKey = key
+	nc := startProviderSubjects(t, cfg, natsURL)
+
+	holder := newTestLeases(t, natsURL, "node-other")
+	_, err := holder.acquire(t.Context(), volumeName)
+	require.NoError(t, err)
+
+	completionSubject, err := ebsprovider.SnapshotCompletionSubject(snapshotID)
+	require.NoError(t, err)
+	completions := make(chan *nats.Msg, 1)
+	sub, err := nc.ChanSubscribe(completionSubject, completions)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, nc.Flush())
+
+	body := marshalRequest(t, map[string]any{
+		"schema_version": ebsprovider.SchemaVersion,
+		"volume_id":      volumeName,
+		"snapshot_id":    snapshotID,
+	})
+	accepted := requestProvider(t, nc, ebsprovider.SnapshotCreateSubjectPrefix+volumeName, body)
+	var acceptance ebsprovider.CreateSnapshotResponse
+	require.NoError(t, json.Unmarshal(accepted.Data, &acceptance))
+	require.Nil(t, acceptance.Error)
+
+	select {
+	case msg := <-completions:
+		var completion ebsprovider.CreateSnapshotResponse
+		require.NoError(t, json.Unmarshal(msg.Data, &completion))
+		require.NotNil(t, completion.Error, "a snapshot under another node's lease must be refused")
+		assert.Equal(t, ebsprovider.ErrorCodeUnavailable, completion.Error.Code)
+		assert.Nil(t, completion.Snapshot)
+	case <-time.After(10 * time.Second):
+		t.Fatal("no completion published")
+	}
+	assert.Empty(t, backend.matching(func(r string) bool { return strings.HasPrefix(r, "PUT ") && strings.Contains(r, snapshotID) }),
+		"no snapshot may be written while another node holds the volume")
+}

spinifex/providers/ebs/viperblock/provider_handlers.go

@@ -149,6 +149,16 @@ func RegisterProviderSubjects(cfg *Config, nc *nats.Conn) error {
 		return fmt.Errorf("subscribe to %s: %w", unsealedSubject, err)
 	}
 
+	// The queue subject is answered by any node's provider; this one only by
+	// this node's, so a daemon can learn what its own provider serves.
+	nodeCapabilitiesSubject, err := ebsprovider.NodeCapabilitiesSubject(cfg.NodeName)
+	if err != nil {
+		return fmt.Errorf("build node capabilities subject: %w", err)
+	}
+	if _, err := nc.Subscribe(nodeCapabilitiesSubject, tracedProviderHandler(handleProviderCapabilities)); err != nil {
+		return fmt.Errorf("subscribe to %s: %w", nodeCapabilitiesSubject, err)
+	}
+
 	return nil
 }
 
@@ -188,7 +198,7 @@ func badRequestError(err error) *ebsprovider.ProviderError {
 }
 
 func unsupportedVersionError(got uint16) *ebsprovider.ProviderError {
-	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: fmt.Sprintf("unsupported schema version %d; supported versions are %d and %d", got, ebsprovider.LegacySchemaVersion, ebsprovider.SchemaVersion)}
+	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: fmt.Sprintf("unsupported schema version %d; %s", got, ebsprovider.SupportedVersionsText())}
 }
 
 // respondPublishedVolume selects the exact publish response a caller asked
@@ -199,19 +209,19 @@ func respondPublishedVolume(ctx context.Context, msg *nats.Msg, schemaVersion ui
 		legacy, err := published.AsV1()
 		if err != nil {
 			respondProvider(ctx, msg, ebsprovider.PublishVolumeResponseV1{
-				Versioned: ebsprovider.NewVersioned(),
+				Versioned: ebsprovider.ReplyVersioned(schemaVersion),
 				Error:     &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: err.Error()},
 			})
 			return
 		}
 		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponseV1{
-			Versioned: ebsprovider.NewVersioned(),
+			Versioned: ebsprovider.ReplyVersioned(schemaVersion),
 			Published: legacy,
 		})
 		return
 	}
 	respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{
-		Versioned: ebsprovider.NewLatestVersioned(),
+		Versioned: ebsprovider.ReplyVersioned(schemaVersion),
 		Published: published,
 	})
 }
@@ -232,6 +242,14 @@ func volumeInUseError(format string, args ...any) *ebsprovider.ProviderError {
 	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: fmt.Sprintf(format, args...)}
 }
 
+func heldElsewhereError(format string, args ...any) *ebsprovider.ProviderError {
+	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeHeldElsewhere, Message: fmt.Sprintf(format, args...)}
+}
+
+// errReadOnlyMismatch is a remount asking for the other access mode than the
+// running export's, which is fixed when nbdkit starts.
+var errReadOnlyMismatch = errors.New("already mounted with the other access mode")
+
 func internalError(format string, args ...any) *ebsprovider.ProviderError {
 	return &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: fmt.Sprintf(format, args...)}
 }
@@ -276,18 +294,21 @@ func findMountedVolume(cfg *Config, volumeID string) (MountedVolume, bool) {
 
 func handleProviderCapabilities(ctx context.Context, msg *nats.Msg) {
 	var req ebsprovider.GetCapabilitiesRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.capabilities: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
 		slog.Error("ebs.provider.capabilities: unsupported schema version", "version", req.SchemaVersion)
-		respondProvider(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.GetCapabilitiesResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	respondProvider(ctx, msg, ebsprovider.GetCapabilitiesResponse{
-		Versioned: ebsprovider.NewVersioned(),
+		Versioned:        v,
+		ContractVersions: ebsprovider.SupportedSchemaVersions(),
 		Capabilities: ebsprovider.Capabilities{
 			// Every snapshot freezes the live checkpoint rather than
 			// depending on a guest-coordinated quiesce.
@@ -387,24 +408,26 @@ func listTopLevelPrefixes(ctx context.Context, store objectstore.ObjectStore, bu
 
 func handleListVolumes(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.ListVolumesRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.list: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 
 	ids, err := listVolumePrefixes(ctx, providerObjectStoreFactory(cfg), cfg.Bucket)
 	if err != nil {
 		slog.Error("ebs.provider.volume.list: failed to enumerate volumes", "err", err)
-		respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("enumerate volumes: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.ListVolumesResponse{Versioned: v, Error: internalError("enumerate volumes: %v", err)})
 		return
 	}
 
-	response := ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned()}
+	response := ebsprovider.ListVolumesResponse{Versioned: v}
 	response.Volumes, response.NextToken = ebsprovider.Page(ids, req.StartingToken, int(req.PageSize()),
 		func(id string) ebsprovider.VolumeRef {
 			return ebsprovider.VolumeRef{ID: id, Handle: volumeHandle(id)}
@@ -417,24 +440,26 @@ func handleListVolumes(ctx context.Context, cfg *Config, msg *nats.Msg) {
 // encrypted state, and reading it would mean a fetch per snapshot listed.
 func handleListSnapshots(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.ListSnapshotsRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.snapshot.list: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 
 	ids, err := listSnapshotPrefixes(ctx, providerObjectStoreFactory(cfg), cfg.Bucket)
 	if err != nil {
 		slog.Error("ebs.provider.snapshot.list: failed to enumerate snapshots", "err", err)
-		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("enumerate snapshots: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.ListSnapshotsResponse{Versioned: v, Error: internalError("enumerate snapshots: %v", err)})
 		return
 	}
 
-	response := ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned()}
+	response := ebsprovider.ListSnapshotsResponse{Versioned: v}
 	response.Snapshots, response.NextToken = ebsprovider.Page(ids, req.StartingToken, int(req.PageSize()),
 		func(id string) ebsprovider.SnapshotRef {
 			return ebsprovider.SnapshotRef{ID: id, Handle: snapshotHandle(id)}
@@ -474,40 +499,42 @@ func buildProviderVBConfig(cfg *Config, volumeID string, volumeSizeBytes uint64,
 
 func handleCreateVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.CreateVolumeRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.create: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
 		slog.Error("ebs.provider.volume.create: unsupported schema version", "version", req.SchemaVersion)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.VolumeID) {
 		slog.Error("ebs.provider.volume.create: invalid volume id", "volume", req.VolumeID)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
 		return
 	}
 	if req.CapacityRange.RequiredBytes <= 0 || (req.CapacityRange.LimitBytes > 0 && req.CapacityRange.RequiredBytes > req.CapacityRange.LimitBytes) {
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid capacity range")})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid capacity range")})
 		return
 	}
 	if req.SourceSnapshotID != "" && req.SourceSnapshotVolumeID == "" {
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("source_snapshot_volume_id is required with source_snapshot_id")})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: invalidArgumentError("source_snapshot_volume_id is required with source_snapshot_id")})
 		return
 	}
 	if err := ebsprovider.ValidateSeedData(req.SeedData); err != nil {
 		slog.Error("ebs.provider.volume.create: seed data rejected", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("%v", err)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: invalidArgumentError("%v", err)})
 		return
 	}
 	if int64(len(req.SeedData)) > req.CapacityRange.RequiredBytes {
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("seed data is larger than the requested capacity")})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: invalidArgumentError("seed data is larger than the requested capacity")})
 		return
 	}
 	if len(req.SeedData) > 0 && req.SourceSnapshotID != "" {
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("seed data cannot be combined with a source snapshot")})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: invalidArgumentError("seed data cannot be combined with a source snapshot")})
 		return
 	}
 
@@ -516,16 +543,16 @@ func handleCreateVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	case err == nil:
 		if existing.CapacityBytes != req.CapacityRange.RequiredBytes {
 			slog.Error("ebs.provider.volume.create: volume exists with different capacity", "volume", req.VolumeID)
-			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: alreadyExistsError("volume %s already exists with a different capacity", req.VolumeID)})
+			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: alreadyExistsError("volume %s already exists with a different capacity", req.VolumeID)})
 			return
 		}
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: existing})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Volume: existing})
 		return
 	case errors.Is(err, viperblock.ErrStateNotFound):
 		// Volume does not exist yet: fall through and create it.
 	default:
 		slog.Error("ebs.provider.volume.create: existence check failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("check existing volume: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: internalError("check existing volume: %v", err)})
 		return
 	}
 
@@ -534,7 +561,7 @@ func handleCreateVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	vb, err := viperblock.New(vbconfig, "s3", s3cfg)
 	if err != nil {
 		slog.Error("ebs.provider.volume.create: new viperblock failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("new viperblock: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: internalError("new viperblock: %v", err)})
 		return
 	}
 	defer vb.Detach()
@@ -542,26 +569,26 @@ func handleCreateVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 
 	if err := vb.Backend.InitCtx(ctx); err != nil {
 		slog.Error("ebs.provider.volume.create: backend init failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("backend init: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: internalError("backend init: %v", err)})
 		return
 	}
 	if req.SourceSnapshotID != "" {
 		if perr := verifySourceSnapshot(vb, req.SourceSnapshotID, req.SourceSnapshotVolumeID); perr != nil {
 			slog.Error("ebs.provider.volume.create: source snapshot rejected", "volume", req.VolumeID, "snapshot", req.SourceSnapshotID, "err", perr.Message)
-			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: perr})
+			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: perr})
 			return
 		}
 	}
 	if len(req.SeedData) > 0 {
 		if err := seedVolume(ctx, vb, req.SeedData); err != nil {
 			slog.Error("ebs.provider.volume.create: seed failed", "volume", req.VolumeID, "bytes", len(req.SeedData), "err", err)
-			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("seed volume: %v", err)})
+			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: internalError("seed volume: %v", err)})
 			return
 		}
 	} else {
 		if err := vb.SaveStateCtx(ctx); err != nil {
 			slog.Error("ebs.provider.volume.create: save state failed", "volume", req.VolumeID, "err", err)
-			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("save state: %v", err)})
+			respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{Versioned: v, Error: internalError("save state: %v", err)})
 			return
 		}
 		// A state read prefers a local copy over the backend, so state left on
@@ -574,7 +601,7 @@ func handleCreateVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 
 	slog.Info("ebs.provider.volume.create: created", "volume", req.VolumeID, "capacityBytes", req.CapacityRange.RequiredBytes, "seedBytes", len(req.SeedData))
 	respondProvider(ctx, msg, ebsprovider.CreateVolumeResponse{
-		Versioned: ebsprovider.NewVersioned(),
+		Versioned: v,
 		Volume: &ebsprovider.Volume{
 			ID:            req.VolumeID,
 			CapacityBytes: req.CapacityRange.RequiredBytes,
@@ -667,72 +694,76 @@ func describeVolumeEngine(ctx context.Context, cfg *Config, volumeID string) (*e
 
 func handleGetVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.GetVolumeRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.describe: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.VolumeID) {
-		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
 		return
 	}
 
 	volume, err := describeVolumeEngine(ctx, cfg, req.VolumeID)
 	if err != nil {
 		if errors.Is(err, viperblock.ErrStateNotFound) {
-			respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: notFoundError("volume %s not found", req.VolumeID)})
+			respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: notFoundError("volume %s not found", req.VolumeID)})
 			return
 		}
 		slog.Error("ebs.provider.volume.describe: failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("describe volume: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Error: internalError("describe volume: %v", err)})
 		return
 	}
-	respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: ebsprovider.NewVersioned(), Volume: volume})
+	respondProvider(ctx, msg, ebsprovider.GetVolumeResponse{Versioned: v, Volume: volume})
 }
 
 func handleExpandVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.ExpandVolumeRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.expand: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.VolumeID) {
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
 		return
 	}
 	if req.CapacityRange.RequiredBytes <= 0 {
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid capacity range")})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid capacity range")})
 		return
 	}
 
 	if mv, ok := findMountedVolume(cfg, req.VolumeID); ok && mv.VB != nil {
 		slog.Error("ebs.provider.volume.expand: volume is mounted", "volume", req.VolumeID)
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is mounted; detach before expanding", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: volumeInUseError("volume %s is mounted; detach before expanding", req.VolumeID)})
 		return
 	}
 
 	vb, lease, err := openVolumeVB(ctx, cfg, req.VolumeID)
 	if err != nil {
 		if errors.Is(err, viperblock.ErrStateNotFound) {
-			respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: notFoundError("volume %s not found", req.VolumeID)})
+			respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: notFoundError("volume %s not found", req.VolumeID)})
 			return
 		}
 		if errors.Is(err, errVolumeLeaseHeld) {
 			slog.Error("ebs.provider.volume.expand: volume is leased elsewhere", "volume", req.VolumeID, "err", err)
-			respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is open elsewhere: %v", req.VolumeID, err)})
+			respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: volumeInUseError("volume %s is open elsewhere: %v", req.VolumeID, err)})
 			return
 		}
 		slog.Error("ebs.provider.volume.expand: open volume failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("open volume: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: internalError("open volume: %v", err)})
 		return
 	}
 	defer func() {
@@ -742,7 +773,7 @@ func handleExpandVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 
 	currentBytes := safecast.Uint64ToInt64(vb.GetVolumeSize())
 	if req.CapacityRange.RequiredBytes < currentBytes {
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("volume expansion is grow-only")})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: invalidArgumentError("volume expansion is grow-only")})
 		return
 	}
 
@@ -751,18 +782,18 @@ func handleExpandVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	rawConfig, err := json.Marshal(vc)
 	if err != nil {
 		slog.Error("ebs.provider.volume.expand: marshal volume config failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("marshal volume config: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: internalError("marshal volume config: %v", err)})
 		return
 	}
 	if err := applyConfigUpdate(ctx, vb, rawConfig); err != nil {
 		slog.Error("ebs.provider.volume.expand: apply config update failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("apply config update: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{Versioned: v, Error: internalError("apply config update: %v", err)})
 		return
 	}
 
 	slog.Info("ebs.provider.volume.expand: expanded", "volume", req.VolumeID, "capacityBytes", req.CapacityRange.RequiredBytes)
 	respondProvider(ctx, msg, ebsprovider.ExpandVolumeResponse{
-		Versioned: ebsprovider.NewVersioned(),
+		Versioned: v,
 		Volume: &ebsprovider.Volume{
 			ID:            req.VolumeID,
 			CapacityBytes: safecast.Uint64ToInt64(vb.GetVolumeSize()),
@@ -856,17 +887,19 @@ func deleteObjectPrefix(ctx context.Context, store objectstore.ObjectStore, buck
 
 func handleDeleteVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *nats.Msg) {
 	var req ebsprovider.DeleteVolumeRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.delete: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.VolumeID) {
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
 		return
 	}
 
@@ -880,7 +913,7 @@ func handleDeleteVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *na
 	}
 	if mountedHere || ownership == ownershipMounted {
 		slog.Error("ebs.provider.volume.delete: volume is published", "volume", req.VolumeID)
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is published; unpublish it before deleting", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: volumeInUseError("volume %s is published; unpublish it before deleting", req.VolumeID)})
 		return
 	}
 	// Ownership could not be established. Deleting is irreversible, so an
@@ -888,7 +921,7 @@ func handleDeleteVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *na
 	// retry once the owner is reachable.
 	if ownership == ownershipUnknown {
 		slog.Error("ebs.provider.volume.delete: could not establish whether the volume is published", "volume", req.VolumeID)
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unavailableError("volume %s: could not establish whether it is published; retry when its owner is reachable", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: unavailableError("volume %s: could not establish whether it is published; retry when its owner is reachable", req.VolumeID)})
 		return
 	}
 
@@ -908,44 +941,46 @@ func handleDeleteVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *na
 	store := providerObjectStoreFactory(cfg)
 	if err := deleteObjectPrefix(ctx, store, cfg.Bucket, req.VolumeID+"-efi/"); err != nil {
 		slog.Error("ebs.provider.volume.delete: failed to delete aux objects", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("delete aux objects: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: internalError("delete aux objects: %v", err)})
 		return
 	}
 	if err := deleteObjectPrefix(ctx, store, cfg.Bucket, req.VolumeID+"/"); err != nil {
 		slog.Error("ebs.provider.volume.delete: failed to delete volume objects", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("delete volume objects: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v, Error: internalError("delete volume objects: %v", err)})
 		return
 	}
 
 	slog.Info("ebs.provider.volume.delete: deleted", "volume", req.VolumeID)
-	respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: ebsprovider.NewVersioned()})
+	respondProvider(ctx, msg, ebsprovider.DeleteVolumeResponse{Versioned: v})
 }
 
 func handleDeleteSnapshot(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.DeleteSnapshotRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.snapshot.delete: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.SnapshotID) {
-		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid snapshot id %q", req.SnapshotID)})
+		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: invalidArgumentError("invalid snapshot id %q", req.SnapshotID)})
 		return
 	}
 
 	store := providerObjectStoreFactory(cfg)
 	if err := deleteObjectPrefix(ctx, store, cfg.Bucket, req.SnapshotID+"/"); err != nil {
 		slog.Error("ebs.provider.snapshot.delete: failed", "snapshot", req.SnapshotID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("delete snapshot objects: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v, Error: internalError("delete snapshot objects: %v", err)})
 		return
 	}
 
 	slog.Info("ebs.provider.snapshot.delete: deleted", "snapshot", req.SnapshotID)
-	respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: ebsprovider.NewVersioned()})
+	respondProvider(ctx, msg, ebsprovider.DeleteSnapshotResponse{Versioned: v})
 }
 
 // handleCopySnapshot serves ebs.provider.v1.snapshot.copy. Unlike
@@ -955,21 +990,23 @@ func handleDeleteSnapshot(ctx context.Context, cfg *Config, msg *nats.Msg) {
 // pattern.
 func handleCopySnapshot(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.CopySnapshotRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.snapshot.copy: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.SourceSnapshotID) || !validVolumeName(req.DestinationSnapshotID) || !validVolumeName(req.VolumeID) {
-		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid source snapshot, destination snapshot, or volume id")})
+		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: invalidArgumentError("invalid source snapshot, destination snapshot, or volume id")})
 		return
 	}
 	if req.SourceSnapshotID == req.DestinationSnapshotID {
-		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("destination snapshot %q must differ from the source", req.DestinationSnapshotID)})
+		respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: invalidArgumentError("destination snapshot %q must differ from the source", req.DestinationSnapshotID)})
 		return
 	}
 
@@ -977,20 +1014,20 @@ func handleCopySnapshot(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	if err != nil {
 		switch {
 		case errors.Is(err, errSnapshotDestinationExists):
-			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: alreadyExistsError("snapshot %s already exists", req.DestinationSnapshotID)})
+			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: alreadyExistsError("snapshot %s already exists", req.DestinationSnapshotID)})
 		case errors.Is(err, viperblock.ErrSnapshotVolumeMismatch):
-			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("%v", err)})
+			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: invalidArgumentError("%v", err)})
 		case errors.Is(err, viperblock.ErrStateNotFound), errors.Is(err, os.ErrNotExist):
-			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: notFoundError("source snapshot %s not found on volume %s", req.SourceSnapshotID, req.VolumeID)})
+			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: notFoundError("source snapshot %s not found on volume %s", req.SourceSnapshotID, req.VolumeID)})
 		default:
 			slog.Error("ebs.provider.snapshot.copy: failed", "volume", req.VolumeID, "source", req.SourceSnapshotID, "destination", req.DestinationSnapshotID, "err", err)
-			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("copy snapshot: %v", err)})
+			respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Error: internalError("copy snapshot: %v", err)})
 		}
 		return
 	}
 
 	slog.Info("ebs.provider.snapshot.copy: copied", "volume", req.VolumeID, "source", req.SourceSnapshotID, "destination", req.DestinationSnapshotID)
-	respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: ebsprovider.NewVersioned(), Snapshot: snapshot})
+	respondProvider(ctx, msg, ebsprovider.CopySnapshotResponse{Versioned: v, Snapshot: snapshot})
 }
 
 // snapshotConfigExists probes whether snapshotID's config object is already
@@ -1055,13 +1092,15 @@ func copySnapshotMetaWithVB(vb *viperblock.VB, volumeID, srcSnapshotID, dstSnaps
 
 func handleCreateSnapshot(ctx context.Context, cfg *Config, nc *nats.Conn, msg *nats.Msg) {
 	var req ebsprovider.CreateSnapshotRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.snapshot.create: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 
@@ -1076,23 +1115,23 @@ func handleCreateSnapshot(ctx context.Context, cfg *Config, nc *nats.Conn, msg *
 	}
 	if !validVolumeName(subjectVolumeID) || subjectVolumeID != req.VolumeID {
 		slog.Error("ebs.provider.snapshot.create: subject/volume mismatch", "subject", msg.Subject, "volume", req.VolumeID)
-		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("volume id %q in subject does not match request volume id %q", subjectVolumeID, req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: invalidArgumentError("volume id %q in subject does not match request volume id %q", subjectVolumeID, req.VolumeID)})
 		return
 	}
 	if !validVolumeName(req.SnapshotID) {
-		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid snapshot id %q", req.SnapshotID)})
+		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: invalidArgumentError("invalid snapshot id %q", req.SnapshotID)})
 		return
 	}
 
 	completionSubject, err := ebsprovider.SnapshotCompletionSubject(req.SnapshotID)
 	if err != nil {
-		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("%s", err.Error())})
+		respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{Versioned: v, Error: invalidArgumentError("%s", err.Error())})
 		return
 	}
 
 	operationID := awsidentifiers.GenerateResourceID("op")
 	respondProvider(ctx, msg, ebsprovider.CreateSnapshotResponse{
-		Versioned:         ebsprovider.NewVersioned(),
+		Versioned:         v,
 		OperationID:       operationID,
 		CompletionSubject: completionSubject,
 		Snapshot: &ebsprovider.Snapshot{
@@ -1117,7 +1156,7 @@ func handleCreateSnapshot(ctx context.Context, cfg *Config, nc *nats.Conn, msg *
 // contract NATSProvider.CreateSnapshot waits on.
 func completeCreateSnapshot(ctx context.Context, cfg *Config, nc *nats.Conn, req ebsprovider.CreateSnapshotRequest, operationID, completionSubject string) {
 	completion := ebsprovider.CreateSnapshotResponse{
-		Versioned:         ebsprovider.NewVersioned(),
+		Versioned:         ebsprovider.ReplyVersioned(req.SchemaVersion),
 		OperationID:       operationID,
 		CompletionSubject: completionSubject,
 	}
@@ -1396,19 +1435,20 @@ func mountVolume(ctx context.Context, cfg *Config, nc *nats.Conn, volumeName str
 	// ebs.mount subject is the route production actually takes: a retried
 	// attach racing a fresh one would otherwise start two real exports.
 	if mv, ok := findMountedVolume(cfg, volumeName); ok {
-		// An entry whose socket is gone describes an export that no longer
-		// exists, and returning its URI makes the volume unstartable forever:
-		// every caller waits for a socket nothing will create. Dropping it and
-		// mounting again is safe because an absent socket cannot be serving.
+		// Returning a dead entry's URI makes the volume unstartable forever:
+		// every caller waits for a socket nothing will create. The entry is
+		// dropped and the volume mounted again once its nbdkit is gone.
 		if stale, why := mountEntryIsStale(mv); stale {
 			slog.WarnContext(ctx, "ebs.mount: dropping stale mount entry and remounting",
 				"volume", volumeName, "uri", mv.NBDURI, "reason", why)
-			releaseStaleMount(ctx, cfg, volumeName)
+			if err := releaseStaleMount(ctx, cfg, volumeName); err != nil {
+				return viperblocklegacyv1.EBSMountResponse{Error: err.Error()}, err
+			}
 		} else {
 			// Access mode is fixed when nbdkit starts, so a remount asking for
 			// the other mode cannot be answered with the running export.
 			if mv.ReadOnly != readOnly {
-				err := fmt.Errorf("volume %s is already mounted read_only=%t on this node", volumeName, mv.ReadOnly)
+				err := fmt.Errorf("volume %s is already mounted read_only=%t on this node: %w", volumeName, mv.ReadOnly, errReadOnlyMismatch)
 				return viperblocklegacyv1.EBSMountResponse{Error: err.Error()}, err
 			}
 			slog.InfoContext(ctx, "ebs.mount: already mounted, returning existing export", "volume", volumeName, "uri", mv.NBDURI)
@@ -1675,13 +1715,13 @@ func unmountVolume(ctx context.Context, cfg *Config, volumeName string) (viperbl
 		// drains to the backend and leaves a receipt. SIGKILLing into that
 		// discards the drain and leaves the WAL for the fallback seal below to
 		// replay, which costs far more than the caller's deadline allows.
-		hostprocess.TerminateProcess(matched.PID, pluginSealGrace)
+		hostprocess.TerminateProcess(matched.PID, cfg.exportSealGrace())
 
 		// The seal below rewrites the directory nbdkit writes, so a kill that
 		// did not take makes it a concurrent write to that directory. Fail the
 		// unmount instead: the entry stays mounted and a retry re-attempts
 		// both, which is what a failed seal already does.
-		if err := hostprocess.ForceKillProcess(matched.PID, fenceKillTimeout); err != nil {
+		if err := hostprocess.ForceKillProcess(matched.PID, cfg.exportKillWait()); err != nil {
 			slog.ErrorContext(ctx, "ebs.unmount: nbdkit did not exit, refusing to seal underneath a live writer",
 				"volume", matched.Name, "pid", matched.PID, "err", err)
 			ebsResponse.Error = fmt.Sprintf("kill nbdkit for %s: %v", matched.Name, err)
@@ -1777,54 +1817,27 @@ func unmountVolume(ctx context.Context, cfg *Config, volumeName string) (viperbl
 // unlike the legacy handler there is no queue-group fallback to reason about.
 func handlePublishVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *nats.Msg) {
 	var req ebsprovider.PublishVolumeRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.publish: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
 		slog.Error("ebs.provider.volume.publish: unsupported schema version", "version", req.SchemaVersion)
-		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	if !validVolumeName(req.VolumeID) {
-		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
 		return
 	}
-	// Idempotent republish: a volume this node already has mounted must not
-	// start a second nbdkit against it (the double-writer hazard this
-	// provider boundary exists to prevent). Match memory.go's behaviour.
-	if mv, ok := findMountedVolume(cfg, req.VolumeID); ok {
-		// Access mode is fixed when nbdkit starts, so a republish asking for
-		// the other mode cannot be answered with the running export.
-		if mv.ReadOnly != req.ReadOnly {
-			respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is already published read_only=%t on this node", req.VolumeID, mv.ReadOnly)})
-			return
-		}
-		slog.Info("ebs.provider.volume.publish: already published, returning existing attachment", "volume", req.VolumeID)
-		respondPublishedVolume(ctx, msg, req.SchemaVersion, &ebsprovider.PublishedVolume{
-			VolumeID: req.VolumeID,
-			NodeID:   cfg.NodeName,
-			Export:   ebsprovider.NewNBDExport(mv.NBDURI),
-		})
-		return
-	}
-
+	// mountVolume returns a live entry's export, and drops and remounts one
+	// whose export has died, so a republish never hands back a dead URI.
 	mountResp, err := mountVolume(ctx, cfg, nc, req.VolumeID, req.ReadOnly)
 	if err != nil {
-		if errors.Is(err, viperblock.ErrStateNotFound) {
-			respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: notFoundError("volume %s not found", req.VolumeID)})
-			return
-		}
-		// Another node holds the volume's lease. This is the exclusion working,
-		// not a failure: reporting it as internal tells the caller something
-		// broke when the answer is that someone else has the volume.
-		if errors.Is(err, errVolumeLeaseHeld) {
-			respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: volumeInUseError("volume %s is held by another node: %v", req.VolumeID, err)})
-			return
-		}
-		slog.Error("ebs.provider.volume.publish: mount failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("mount volume: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.PublishVolumeResponse{Versioned: v, Error: publishError(req.SchemaVersion, req.VolumeID, err)})
 		return
 	}
 
@@ -1836,59 +1849,110 @@ func handlePublishVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *n
 	})
 }
 
+// publishError classifies a failed mount for the contract reply. A lease held
+// by another node is the exclusion working, and a backing store that is not
+// ready yet is worth retrying, so neither is reported as internal.
+func publishError(version uint16, volumeID string, err error) *ebsprovider.ProviderError {
+	switch {
+	case errors.Is(err, viperblock.ErrStateNotFound):
+		return notFoundError("volume %s not found", volumeID)
+	case errors.Is(err, errVolumeLeaseHeld) && version >= ebsprovider.HeldElsewhereSchemaVersion:
+		return heldElsewhereError("volume %s is held by another node: %v", volumeID, err)
+	case errors.Is(err, errVolumeLeaseHeld):
+		// Below version 3 the refusal keeps the code v1.23.0 sends.
+		return volumeInUseError("volume %s is held by another node: %v", volumeID, err)
+	case errors.Is(err, errReadOnlyMismatch):
+		return volumeInUseError("%v", err)
+	case mountErrRetryable(err), errors.Is(err, ebsprovider.ErrUnavailable):
+		return unavailableError("mount volume %s: %v", volumeID, err)
+	default:
+		slog.Error("ebs.provider.volume.publish: mount failed", "volume", volumeID, "err", err)
+		return internalError("mount volume: %v", err)
+	}
+}
+
 // handleUnpublishVolume serves ebs.provider.v1.<node>.unmount, fronting the
 // same seal-decision path ebs.unmount uses. A failed seal is reported as an
 // error while leaving the volume in cfg.MountedVolumes (see unmountVolume),
 // never as success.
 func handleUnpublishVolume(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.UnpublishVolumeRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
 		slog.Error("ebs.provider.volume.unpublish: bad request", "err", err)
-		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
-	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		slog.Error("ebs.provider.volume.unpublish: unsupported schema version", "version", req.SchemaVersion)
-		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+	if err := ebsprovider.CheckUnpublishVersion(req); err != nil {
+		slog.Error("ebs.provider.volume.unpublish: request refused", "version", req.SchemaVersion, "no_seal", req.NoSeal, "err", err)
+		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: unpublishRequestError(req.SchemaVersion, err)})
 		return
 	}
 	if !validVolumeName(req.VolumeID) {
-		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
+		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", req.VolumeID)})
 		return
 	}
 
+	tornDown, err := unpublishVolume(ctx, cfg, req)
+	if err != nil {
+		slog.Error("ebs.provider.volume.unpublish: unmount failed", "volume", req.VolumeID, "no_seal", req.NoSeal, "err", err)
+		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, Error: internalError("unmount volume: %v", err)})
+		return
+	}
+
+	slog.Info("ebs.provider.volume.unpublish: unpublished", "volume", req.VolumeID, "node", cfg.NodeName, "no_seal", req.NoSeal, "torn_down", tornDown)
+	respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: v, TornDown: tornDown})
+}
+
+// unpublishRequestError classifies a CheckUnpublishVersion refusal.
+func unpublishRequestError(version uint16, err error) *ebsprovider.ProviderError {
+	if errors.Is(err, ebsprovider.ErrUnsupportedVersion) {
+		return unsupportedVersionError(version)
+	}
+	return invalidArgumentError("%v", err)
+}
+
+// unpublishVolume tears down this node's export and reports whether there was
+// one, which is only reported from NoSealSchemaVersion on. NoSeal drops it
+// unsealed, as an abandon does; otherwise it seals first, as ebs.unmount does.
+func unpublishVolume(ctx context.Context, cfg *Config, req ebsprovider.UnpublishVolumeRequest) (bool, error) {
+	tornDown, err := tearDownPublication(ctx, cfg, req)
+	return tornDown && req.SchemaVersion >= ebsprovider.NoSealSchemaVersion, err
+}
+
+func tearDownPublication(ctx context.Context, cfg *Config, req ebsprovider.UnpublishVolumeRequest) (bool, error) {
+	if req.NoSeal {
+		return cfg.abandonVolume(ctx, req.VolumeID, "unpublish with no_seal")
+	}
 	unmountResp, err := unmountVolume(ctx, cfg, req.VolumeID)
 	if err != nil {
 		// Nothing mounted here is already the state unpublish asks for, so a
 		// retry after a request that completed server-side converges rather
 		// than erroring, matching the legacy path's unmountResponseError.
 		if unmountResp.NotFound {
-			slog.Info("ebs.provider.volume.unpublish: already unpublished", "volume", req.VolumeID, "node", cfg.NodeName)
-			respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned()})
-			return
+			return unmountResp.Reaped, nil
 		}
-		slog.Error("ebs.provider.volume.unpublish: unmount failed", "volume", req.VolumeID, "err", err)
-		respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("unmount volume: %v", err)})
-		return
+		return false, err
 	}
-
-	slog.Info("ebs.provider.volume.unpublish: unpublished", "volume", req.VolumeID, "node", cfg.NodeName)
-	respondProvider(ctx, msg, ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.NewVersioned()})
+	return true, nil
 }
 
 func handleHoldsUnsealedWrites(ctx context.Context, cfg *Config, msg *nats.Msg) {
 	var req ebsprovider.HoldsUnsealedWritesRequest
-	if err := json.Unmarshal(msg.Data, &req); err != nil {
-		respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: ebsprovider.NewVersioned(), Error: badRequestError(err)})
+	err := json.Unmarshal(msg.Data, &req)
+	v := ebsprovider.ReplyVersioned(req.SchemaVersion)
+	if err != nil {
+		respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: badRequestError(err)})
 		return
 	}
 	if !ebsprovider.SupportsSchemaVersion(req.SchemaVersion) {
-		respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: ebsprovider.NewVersioned(), Error: unsupportedVersionError(req.SchemaVersion)})
+		respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: unsupportedVersionError(req.SchemaVersion)})
 		return
 	}
 	for _, volume := range req.VolumeIDs {
 		if !validVolumeName(volume) {
-			respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: ebsprovider.NewVersioned(), Error: invalidArgumentError("invalid volume id %q", volume)})
+			respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: invalidArgumentError("invalid volume id %q", volume)})
 			return
 		}
 	}
@@ -1896,19 +1960,18 @@ func handleHoldsUnsealedWrites(ctx context.Context, cfg *Config, msg *nats.Msg)
 	held, err := cfg.unsealedWritesHeld(ctx, req.VolumeIDs)
 	if err != nil {
 		slog.Error("ebs.provider.unsealed: could not read dirty markers", "node", cfg.NodeName, "err", err)
-		respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: ebsprovider.NewVersioned(), Error: internalError("read dirty markers: %v", err)})
+		respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Error: internalError("read dirty markers: %v", err)})
 		return
 	}
-	respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: ebsprovider.NewVersioned(), Held: held})
+	respondProvider(ctx, msg, ebsprovider.HoldsUnsealedWritesResponse{Versioned: v, Held: held})
 }
 
-// mountEntryIsStale reports whether a MountedVolume describes an export that is
-// no longer running, and why.
+// mountEntryIsStale reports whether a MountedVolume describes an export that no
+// new client can reach, and why.
 //
-// Only positive evidence of death counts. The entry guards against a second
-// nbdkit for one volume, so treating a live export as stale would create the
-// double-writer hazard it exists to prevent — an absent socket or a dead pid
-// are the two conditions under which no export can be serving.
+// Only positive evidence counts: an absent socket or a dead pid. An absent
+// socket does not mean the nbdkit has exited, since a client that connected
+// before it went keeps writing through it, so releaseStaleMount stops it.
 func mountEntryIsStale(mv MountedVolume) (bool, string) {
 	if mv.Socket != "" {
 		if _, err := os.Stat(mv.Socket); errors.Is(err, os.ErrNotExist) {
@@ -1923,7 +1986,18 @@ func mountEntryIsStale(mv MountedVolume) (bool, string) {
 
 // releaseStaleMount drops a dead export's entry and everything it held, so the
 // remount that follows starts from the same state a clean unmount would leave.
-func releaseStaleMount(ctx context.Context, cfg *Config, volumeName string) {
+//
+// An nbdkit still running behind a lost socket is a writer, so it goes through
+// unmountVolume, which stops it and seals its copy. If it will not stop, the
+// entry stays and the error wraps ErrUnavailable: a remount would be a second writer.
+func releaseStaleMount(ctx context.Context, cfg *Config, volumeName string) error {
+	if mv, ok := findMountedVolume(cfg, volumeName); ok && hostprocess.ProcessAlive(mv.PID) {
+		if _, err := unmountVolume(ctx, cfg, volumeName); err != nil {
+			return fmt.Errorf("%w: stop nbdkit %d behind the lost socket of %s: %w", ebsprovider.ErrUnavailable, mv.PID, volumeName, err)
+		}
+		return nil
+	}
+
 	cfg.mu.Lock()
 	var matched MountedVolume
 	found := false
@@ -1936,7 +2010,7 @@ func releaseStaleMount(ctx context.Context, cfg *Config, volumeName string) {
 	}
 	cfg.mu.Unlock()
 	if !found {
-		return
+		return nil
 	}
 
 	if matched.ConfigSub != nil {
@@ -1961,4 +2035,5 @@ func releaseStaleMount(ctx context.Context, cfg *Config, volumeName string) {
 		}
 	}
 	cfg.releaseVolumeLease(ctx, matched.Lease)
+	return nil
 }

spinifex/providers/ebs/viperblock/provider_handlers_test.go

@@ -150,6 +150,11 @@ func providerSubjectCases() []providerSubjectCase {
 			"volume_id": "vol-testunpublish01",
 			"node_id":   "test-node",
 		}},
+		{"holds_unsealed_writes", "ebs.provider.v1.test-node.unsealed", map[string]any{
+			"node_id":    "test-node",
+			"volume_ids": []string{"vol-testunsealed01"},
+		}},
+		{"node_capabilities", "ebs.provider.v1.test-node.capabilities", map[string]any{}},
 	}
 }
 
@@ -867,7 +872,7 @@ func TestProviderHandlers_PublishVolume_IdempotentRepublish(t *testing.T) {
 	cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{
 		Name:   volumeName,
 		VB:     vb,
-		PID:    424242,
+		PID:    os.Getpid(),
 		NBDURI: existingURI,
 	})
 
@@ -893,7 +898,7 @@ func TestProviderHandlers_PublishVolume_IdempotentRepublish(t *testing.T) {
 	pid := cfg.MountedVolumes[0].PID
 	cfg.mu.Unlock()
 	assert.Equal(t, 1, count, "republish of an already-mounted volume must not add a second entry")
-	assert.Equal(t, 424242, pid, "republish must not spawn a second nbdkit (PID must be unchanged)")
+	assert.Equal(t, os.Getpid(), pid, "republish must not spawn a second nbdkit (PID must be unchanged)")
 }
 
 func TestProviderHandlers_PublishVolume_LegacyV1Wire(t *testing.T) {
@@ -905,7 +910,7 @@ func TestProviderHandlers_PublishVolume_LegacyV1Wire(t *testing.T) {
 	const existingURI = "nbd:unix:/fake/vol-publegacyv1.sock"
 	vb := createTestVBWithState(t, volumeName)
 	cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{
-		Name: volumeName, VB: vb, PID: 424242, NBDURI: existingURI,
+		Name: volumeName, VB: vb, PID: os.Getpid(), NBDURI: existingURI,
 	})
 
 	publishSubject, err := ebsprovider.PublishSubject(cfg.NodeName)
@@ -939,7 +944,7 @@ func TestProviderHandlers_PublishVolume_ReadOnlyModeMismatchRefused(t *testing.T
 	cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{
 		Name:     volumeName,
 		VB:       vb,
-		PID:      424243,
+		PID:      os.Getpid(),
 		NBDURI:   "nbd:unix:/fake/vol-readonlypub001.sock",
 		ReadOnly: false,
 	})
@@ -1038,8 +1043,8 @@ func TestProviderHandlers_PublishUnpublish_RequireNodeName(t *testing.T) {
 	cfgNamed := setupTestConfig(t, natsURL2)
 	ncNamed := startProviderSubjects(t, cfgNamed, natsURL2)
 
-	assert.Equal(t, ncNamed.NumSubscriptions()-3, ncNoNode.NumSubscriptions(),
-		"an empty NodeName must skip exactly the publish, unpublish and unsealed-writes subscriptions")
+	assert.Equal(t, ncNamed.NumSubscriptions()-4, ncNoNode.NumSubscriptions(),
+		"an empty NodeName must skip exactly the publish, unpublish, unsealed-writes and node capabilities subscriptions")
 
 	// The other provider subjects must still work without a node name.
 	body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion})

spinifex/providers/ebs/viperblock/stale_mount_test.go

@@ -4,12 +4,20 @@ package viperblock
 //exporting them would widen the package API for a test rather than for a caller.
 
 import (
+	"context"
+	"errors"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"testing"
+	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
+	"github.com/mulgadc/viperblock/viperblock"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
 )
 
 // A mount entry pointing at a socket that no longer exists describes an export
@@ -47,11 +55,11 @@ func TestMountEntryIsStaleWhenProcessIsGone(t *testing.T) {
 func TestReleaseStaleMountDropsTheEntry(t *testing.T) {
 	cfg := &Config{MountedVolumes: []MountedVolume{{Name: "vol-1"}, {Name: "vol-2"}}}
 
-	releaseStaleMount(t.Context(), cfg, "vol-1")
+	require.NoError(t, releaseStaleMount(t.Context(), cfg, "vol-1"))
 	require.Len(t, cfg.MountedVolumes, 1)
 	assert.Equal(t, "vol-2", cfg.MountedVolumes[0].Name)
 
-	releaseStaleMount(t.Context(), cfg, "vol-absent")
+	require.NoError(t, releaseStaleMount(t.Context(), cfg, "vol-absent"))
 	assert.Len(t, cfg.MountedVolumes, 1, "releasing an unmounted volume must be a no-op")
 }
 
@@ -64,3 +72,76 @@ func deadPID(t *testing.T) int {
 	require.NoError(t, err)
 	return state.Pid()
 }
+
+// orphanedConfig is fencedConfig with the entry's socket gone and local state
+// under BaseDir, so a stop through the unmount path has a copy to seal.
+func orphanedConfig(t *testing.T, volumeName string, pid int) *Config {
+	t.Helper()
+	_, natsURL := setupEmbeddedNATS(t)
+	cfg := fencedConfig(t, natsURL, "node-a", volumeName)
+	withLocalState(t, cfg, volumeName)
+	cfg.MountedVolumes[0].Socket = filepath.Join(t.TempDir(), "gone.sock")
+	if pid != 0 {
+		cfg.MountedVolumes[0].PID = pid
+	}
+	return cfg
+}
+
+// An nbdkit whose socket is gone still serves a client connected before it
+// went, so a remount beside it is a second writer. It has to be stopped, and
+// its copy sealed as any unmount would, before the volume is opened again.
+func TestMountVolume_SocketGoneProcessAliveIsStoppedBeforeRemount(t *testing.T) {
+	const volumeName = "vol-orphanalive1"
+	cfg := orphanedConfig(t, volumeName, 0)
+	pid := cfg.MountedVolumes[0].PID
+	require.True(t, hostprocess.ProcessAlive(pid))
+
+	sealed := false
+	cfg.sealVolume = func(context.Context, string) error { sealed = true; return nil }
+	errNoRemount := errors.New("remount stops here")
+	var aliveAtRemount []bool
+	cfg.constructVB = func(context.Context, string) (*viperblock.VB, int, error) {
+		aliveAtRemount = append(aliveAtRemount, hostprocess.ProcessAlive(pid))
+		return nil, 0, errNoRemount
+	}
+
+	_, err := mountVolume(t.Context(), cfg, nil, volumeName, false)
+	require.ErrorIs(t, err, errNoRemount, "the remount must be reached once the old nbdkit is stopped")
+	assert.Equal(t, []bool{false}, aliveAtRemount, "the old nbdkit must be gone before the volume is opened again")
+	assert.True(t, sealed, "the old export's copy is sealed as a normal unmount would seal it")
+}
+
+// An nbdkit that will not stop is still a writer, so the publish is refused as
+// retry-later and the entry kept for the next attempt to retry the stop.
+func TestMountVolume_SocketGoneProcessUnstoppableIsUnavailable(t *testing.T) {
+	const volumeName = "vol-orphanstuck1"
+	cfg := orphanedConfig(t, volumeName, zombiePID(t))
+	cfg.exportStopWait = 50 * time.Millisecond
+	sealed := false
+	cfg.sealVolume = func(context.Context, string) error { sealed = true; return nil }
+	cfg.constructVB = func(context.Context, string) (*viperblock.VB, int, error) {
+		t.Error("a volume must not be opened again beside an nbdkit that would not stop")
+		return nil, 0, errors.New("unreachable")
+	}
+
+	_, err := mountVolume(t.Context(), cfg, nil, volumeName, false)
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable)
+	assert.Equal(t, ebsprovider.ErrorCodeUnavailable, publishError(ebsprovider.SchemaVersion, volumeName, err).Code)
+	_, mounted := findMountedVolume(cfg, volumeName)
+	assert.True(t, mounted, "the entry must stay while its nbdkit may still write")
+	assert.False(t, sealed, "nothing may seal underneath a writer that is still running")
+}
+
+// zombiePID returns an exited, unreaped child: kill(pid,0) still reaches it and
+// no signal makes it go, so it stands in for an nbdkit that will not stop.
+func zombiePID(t *testing.T) int {
+	t.Helper()
+	hostprocess.ShortenProcessExitPollForTest(t, 5*time.Millisecond)
+	cmd := exec.Command("sleep", "300")
+	require.NoError(t, cmd.Start())
+	require.NoError(t, cmd.Process.Kill())
+	var info unix.Siginfo
+	require.NoError(t, unix.Waitid(unix.P_PID, cmd.Process.Pid, &info, unix.WEXITED|unix.WNOWAIT, nil))
+	t.Cleanup(func() { _ = cmd.Wait() })
+	return cmd.Process.Pid
+}

spinifex/providers/ebs/viperblock/viperblock.go

@@ -214,6 +214,10 @@ type Config struct {
 	fenceWatchEvery time.Duration
 	processAlive    func(pid int) bool
 
+	// exportStopWait overrides both waits unmountVolume gives nbdkit to exit,
+	// pluginSealGrace and fenceKillTimeout. Zero means those.
+	exportStopWait time.Duration
+
 	// fenceNoticeEvery overrides the wait between fence notice attempts. Zero
 	// means defaultFenceNoticeRetry.
 	fenceNoticeEvery time.Duration
@@ -245,6 +249,20 @@ type Config struct {
 	mu sync.Mutex
 }
 
+func (cfg *Config) exportSealGrace() time.Duration {
+	if cfg.exportStopWait > 0 {
+		return cfg.exportStopWait
+	}
+	return pluginSealGrace
+}
+
+func (cfg *Config) exportKillWait() time.Duration {
+	if cfg.exportStopWait > 0 {
+		return cfg.exportStopWait
+	}
+	return fenceKillTimeout
+}
+
 // seal persists volumeName's block map to predastore, honouring a test's
 // injected seal if there is one.
 func (cfg *Config) seal(ctx context.Context, volumeName string) error {

spinifex/providers/ebs/viperblock/volume_abandon_test.go

@@ -63,6 +63,7 @@ func TestVolumeAbandon_DoesNotSeal(t *testing.T) {
 
 	sealed := false
 	cfg := fencedConfig(t, natsURL, "node-a", volumeName)
+	withLocalState(t, cfg, volumeName)
 	cfg.sealVolume = func(context.Context, string) error { sealed = true; return nil }
 
 	_, err := cfg.abandonVolume(t.Context(), volumeName, "the instance moved")

spinifex/providers/ebs/viperblock/volume_fence_test.go

@@ -5,7 +5,9 @@ package viperblock
 
 import (
 	"context"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -70,6 +72,15 @@ func fencedConfig(t *testing.T, natsURL, owner, volumeName string) *Config {
 	return cfg
 }
 
+// withLocalState gives cfg a BaseDir holding local state for volumeName, so a
+// teardown that seals has a copy to seal and a test that no seal ran means it.
+func withLocalState(t *testing.T, cfg *Config, volumeName string) {
+	t.Helper()
+	cfg.BaseDir = t.TempDir()
+	require.NoError(t, os.MkdirAll(filepath.Join(cfg.BaseDir, volumeName), 0o750))
+	require.True(t, volumeNeedsSeal(volumeName, cfg.BaseDir), "a seal must be reachable for the test to mean anything")
+}
+
 // TestVolumeFence_TearsDownWhenAnotherNodeHoldsTheLease is the property the
 // fence exists for. A node that has lost its lease is a second writer, and the
 // only way to stop being one without a backend that can refuse is to give up

spinifex/runtime/daemon/daemon.go

@@ -1971,6 +1971,17 @@ func (d *Daemon) startCluster() error {
 		return fmt.Errorf("initialize JetStream: %w", err)
 	}
 
+	// Before any service is built: a node on a different provider from the
+	// cluster's would address existing volumes through the wrong implementation.
+	// A refusal exits so the unit shows failed rather than a healthy idle daemon.
+	if err := d.claimEBSProvider(); err != nil {
+		if errors.Is(err, errStartupPermanent) {
+			slog.Error("Refusing to start: EBS provider lock", "err", err)
+			d.exitFunc(1)
+		}
+		return fmt.Errorf("EBS provider lock: %w", err)
+	}
+
 	// Remove the obsolete spinifex-dhcp-leases bucket (idempotent).
 	if js, jsErr := jetstream.New(d.natsConn); jsErr == nil {
 		if err := kvutil.DeleteBucketIfExists(d.ctx, js, "spinifex-dhcp-leases"); err != nil {
@@ -2738,6 +2749,10 @@ func (d *Daemon) initJetStream() error {
 			err = d.jsManager.InitTerminatedInstanceBucket()
 		}
 
+		if err == nil {
+			err = d.jsManager.InitClusterConfigBucket()
+		}
+
 		if err == nil {
 			d.jsManager.SetSyncObserver(d)
 			// Replicas cannot be undeclared here: bucket creation refuses
@@ -2786,6 +2801,29 @@ func (d *Daemon) upgradeJetStreamReplicas() {
 	}
 }
 
+// claimEBSProvider records this node's EBS provider as the cluster's, or
+// refuses when the cluster already records a different one.
+func (d *Daemon) claimEBSProvider() error {
+	provider := d.config.EBS.ResolvedProvider()
+	rev, err := initServiceWithRetry("EBS provider lock", func() (uint64, error) {
+		ctx, cancel := context.WithTimeout(context.Background(), ebsProviderClaimTimeout)
+		defer cancel()
+		return d.jsManager.ClaimEBSProvider(ctx, d.node, provider)
+	})
+	if err != nil {
+		return err
+	}
+	slog.Info("EBS provider matches the cluster's", "provider", provider, "revision", rev)
+	return nil
+}
+
+// ebsProviderClaimTimeout bounds one attempt at the provider lock's KV calls.
+const ebsProviderClaimTimeout = 10 * time.Second
+
+// errStartupPermanent marks an error that a startup retry loop cannot clear,
+// so the loop returns it at once rather than waiting out its budget.
+var errStartupPermanent = errors.New("permanent startup refusal")
+
 // initRetrySleep is the sleep seam used by initServiceWithRetry; tests override it.
 var initRetrySleep = time.Sleep
 
@@ -2806,6 +2844,10 @@ func initServiceWithRetry[T any](name string, initFn func() (T, error)) (T, erro
 			}
 			return result, nil
 		}
+		if errors.Is(err, errStartupPermanent) {
+			var zero T
+			return zero, err
+		}
 
 		elapsed := time.Since(start)
 		if elapsed >= maxWait {

spinifex/runtime/daemon/ebs_provider_lock.go

@@ -0,0 +1,103 @@
+package daemon
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// EBSProviderKey is the ClusterConfigBucket key recording the one EBS
+// provider the cluster runs.
+const EBSProviderKey = "ebs-provider"
+
+// EBSProviderRecord is the value at EBSProviderKey.
+type EBSProviderRecord struct {
+	Provider string `json:"provider"`
+	// RecordedBy names the node whose Create won, for the refusal message.
+	RecordedBy string `json:"recorded_by"`
+}
+
+// EBSProviderMismatchError is the refusal a daemon returns when its configured
+// provider is not the one the cluster recorded. Retrying cannot clear it.
+type EBSProviderMismatchError struct {
+	Node       string
+	Configured string
+	Recorded   EBSProviderRecord
+}
+
+func (e *EBSProviderMismatchError) Error() string {
+	return fmt.Sprintf("EBS provider mismatch: node %s is configured with provider %q, but this cluster runs %q (recorded by node %s in KV bucket %s, key %s); "+
+		"set [nodes.%s.ebs] provider = %q to join it, or change the cluster's provider by wiping and reforming the cluster, since volumes record no provider to migrate by",
+		e.Node, e.Configured, e.Recorded.Provider, e.Recorded.RecordedBy, ClusterConfigBucket, EBSProviderKey, e.Node, e.Recorded.Provider)
+}
+
+func (e *EBSProviderMismatchError) Unwrap() error { return errStartupPermanent }
+
+// EBSProviderRecordInvalidError reports a recorded value that does not decode
+// or names no supported provider, so there is nothing to compare against.
+type EBSProviderRecordInvalidError struct {
+	Raw    []byte
+	Reason string
+}
+
+func (e *EBSProviderRecordInvalidError) Error() string {
+	return fmt.Sprintf("EBS provider lock unreadable: KV bucket %s, key %s holds %q (%s); "+
+		"correct the value to {\"provider\":\"<provider>\"} naming the provider the cluster's volumes were created on, or delete the key so the next daemon records its own",
+		ClusterConfigBucket, EBSProviderKey, e.Raw, e.Reason)
+}
+
+func (e *EBSProviderRecordInvalidError) Unwrap() error { return errStartupPermanent }
+
+// ClaimEBSProvider records provider as the cluster's EBS provider if none is
+// recorded, and otherwise checks it against the recorded one. Create decides
+// a race, so one provider is ever recorded; it returns that record's revision.
+func (m *JetStreamManager) ClaimEBSProvider(ctx context.Context, node, provider string) (uint64, error) {
+	if m.clusterConfig == nil {
+		return 0, errors.New("cluster config KV not initialized")
+	}
+	claim := &EBSProviderRecord{Provider: provider, RecordedBy: node}
+	rev, err := kvstore.On[EBSProviderRecord](m.clusterConfig).Create(ctx, EBSProviderKey, claim)
+	if err == nil {
+		return rev, nil
+	}
+	if !errors.Is(err, kvstore.ErrExists) {
+		return 0, fmt.Errorf("record EBS provider: %w", err)
+	}
+	recorded, rev, err := m.readEBSProvider(ctx)
+	if err != nil {
+		return 0, err
+	}
+	if recorded.Provider != provider {
+		return 0, &EBSProviderMismatchError{Node: node, Configured: provider, Recorded: *recorded}
+	}
+	return rev, nil
+}
+
+// readEBSProvider reads the raw value rather than through the store's codec,
+// so a malformed one can be quoted in the refusal.
+func (m *JetStreamManager) readEBSProvider(ctx context.Context) (*EBSProviderRecord, uint64, error) {
+	kv, err := m.clusterConfig.KV(ctx)
+	if err != nil {
+		return nil, 0, err
+	}
+	entry, err := kv.Get(ctx, EBSProviderKey)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			return nil, 0, fmt.Errorf("recorded EBS provider vanished after Create lost: %w", err)
+		}
+		return nil, 0, fmt.Errorf("read recorded EBS provider: %w", err)
+	}
+	var rec EBSProviderRecord
+	if err := json.Unmarshal(entry.Value(), &rec); err != nil {
+		return nil, 0, &EBSProviderRecordInvalidError{Raw: entry.Value(), Reason: "not a JSON record: " + err.Error()}
+	}
+	if !config.SupportedEBSProvider(rec.Provider) {
+		return nil, 0, &EBSProviderRecordInvalidError{Raw: entry.Value(), Reason: fmt.Sprintf("provider %q is not a supported provider", rec.Provider)}
+	}
+	return &rec, entry.Revision(), nil
+}

spinifex/runtime/daemon/ebs_provider_lock_test.go

@@ -0,0 +1,272 @@
+package daemon
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"slices"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newLockManager opens the cluster-config bucket over its own connection, as
+// a separate daemon would.
+func newLockManager(t *testing.T, url string) *JetStreamManager {
+	t.Helper()
+	nc, err := nats.Connect(url)
+	require.NoError(t, err)
+	t.Cleanup(nc.Close)
+	m, err := NewJetStreamManager(nc)
+	require.NoError(t, err)
+	require.NoError(t, m.InitClusterConfigBucket())
+	return m
+}
+
+func recordedEBSProvider(t *testing.T, js jetstream.JetStream) (EBSProviderRecord, uint64) {
+	t.Helper()
+	kv, err := js.KeyValue(t.Context(), ClusterConfigBucket)
+	require.NoError(t, err)
+	entry, err := kv.Get(t.Context(), EBSProviderKey)
+	require.NoError(t, err)
+	var rec EBSProviderRecord
+	require.NoError(t, json.Unmarshal(entry.Value(), &rec))
+	return rec, entry.Revision()
+}
+
+func TestClusterConfigBucket_NoTTLAndSchemaVersion(t *testing.T) {
+	ns, _, js := testutil.StartTestJetStream(t)
+	newLockManager(t, ns.ClientURL())
+
+	kv, err := js.KeyValue(t.Context(), ClusterConfigBucket)
+	require.NoError(t, err)
+	status, err := kv.Status(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, int64(1), status.History())
+	assert.Zero(t, status.TTL(), "a TTL would expire the lock and let a second provider in")
+
+	v, err := kvutil.ReadVersion(t.Context(), kv)
+	require.NoError(t, err)
+	assert.Equal(t, 1, v)
+}
+
+func TestClaimEBSProvider_FirstClaimRecordsAndMatchingNodesJoin(t *testing.T) {
+	ns, _, js := testutil.StartTestJetStream(t)
+	first := newLockManager(t, ns.ClientURL())
+	second := newLockManager(t, ns.ClientURL())
+
+	rev, err := first.ClaimEBSProvider(t.Context(), "node1", config.EBSProviderQEMUNBD)
+	require.NoError(t, err)
+
+	joined, err := second.ClaimEBSProvider(t.Context(), "node2", config.EBSProviderQEMUNBD)
+	require.NoError(t, err)
+	assert.Equal(t, rev, joined)
+	restarted, err := first.ClaimEBSProvider(t.Context(), "node1", config.EBSProviderQEMUNBD)
+	require.NoError(t, err, "a restart on the same provider must start")
+	assert.Equal(t, rev, restarted)
+
+	rec, revAfter := recordedEBSProvider(t, js)
+	assert.Equal(t, EBSProviderRecord{Provider: config.EBSProviderQEMUNBD, RecordedBy: "node1"}, rec)
+	assert.Equal(t, rev, revAfter, "a matching claim must not rewrite the key")
+}
+
+func TestClaimEBSProvider_DifferentProviderRefused(t *testing.T) {
+	ns, _, js := testutil.StartTestJetStream(t)
+	first := newLockManager(t, ns.ClientURL())
+	second := newLockManager(t, ns.ClientURL())
+
+	_, err := first.ClaimEBSProvider(t.Context(), "node1", config.EBSProviderViperblockd)
+	require.NoError(t, err)
+	_, err = second.ClaimEBSProvider(t.Context(), "node2", config.EBSProviderQEMUNBD)
+
+	var mismatch *EBSProviderMismatchError
+	require.ErrorAs(t, err, &mismatch)
+	assert.ErrorIs(t, err, errStartupPermanent)
+	assert.Equal(t, "node2", mismatch.Node)
+	assert.Equal(t, config.EBSProviderQEMUNBD, mismatch.Configured)
+	assert.Equal(t, EBSProviderRecord{Provider: config.EBSProviderViperblockd, RecordedBy: "node1"}, mismatch.Recorded)
+	for _, want := range []string{`"qemunbd"`, `"viperblockd"`, "node1", ClusterConfigBucket, EBSProviderKey, `[nodes.node2.ebs] provider = "viperblockd"`, "wiping and reforming"} {
+		assert.Contains(t, err.Error(), want)
+	}
+
+	rec, _ := recordedEBSProvider(t, js)
+	assert.Equal(t, config.EBSProviderViperblockd, rec.Provider, "a refused claim must leave the recorded provider in place")
+}
+
+// A value that does not decode, or names no supported provider, is refused as
+// unreadable rather than as a mismatch, and quotes what the key holds.
+func TestClaimEBSProvider_InvalidRecordRefused(t *testing.T) {
+	cases := []struct {
+		name   string
+		raw    string
+		reason string
+	}{
+		{"not JSON", "qemunbd", "not a JSON record"},
+		{"empty provider", `{"provider":"","recorded_by":"node1"}`, `provider "" is not a supported provider`},
+		{"unknown provider", `{"provider":"cephd","recorded_by":"node1"}`, `provider "cephd" is not a supported provider`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			ns, _, js := testutil.StartTestJetStream(t)
+			m := newLockManager(t, ns.ClientURL())
+			kv, err := js.KeyValue(t.Context(), ClusterConfigBucket)
+			require.NoError(t, err)
+			_, err = kv.Put(t.Context(), EBSProviderKey, []byte(tc.raw))
+			require.NoError(t, err)
+
+			_, err = m.ClaimEBSProvider(t.Context(), "node2", config.EBSProviderViperblockd)
+			var invalid *EBSProviderRecordInvalidError
+			require.ErrorAs(t, err, &invalid)
+			assert.ErrorIs(t, err, errStartupPermanent)
+			assert.NotErrorAs(t, err, new(*EBSProviderMismatchError))
+			for _, want := range []string{ClusterConfigBucket, EBSProviderKey, strconv.Quote(tc.raw), tc.reason} {
+				assert.Contains(t, err.Error(), want)
+			}
+		})
+	}
+}
+
+// Two daemons on different providers starting together: whichever Create wins
+// is recorded, and the other is refused.
+func TestClaimEBSProvider_ConcurrentDifferentProvidersLeaveOneKeyAndOneRefusal(t *testing.T) {
+	for range 20 {
+		ns, _, js := testutil.StartTestJetStream(t)
+		managers := []*JetStreamManager{newLockManager(t, ns.ClientURL()), newLockManager(t, ns.ClientURL())}
+		providers := []string{config.EBSProviderViperblockd, config.EBSProviderQEMUNBD}
+		nodes := []string{"node1", "node2"}
+
+		errs := make([]error, 2)
+		revs := make([]uint64, 2)
+		start := make(chan struct{})
+		var wg sync.WaitGroup
+		for i := range managers {
+			wg.Go(func() {
+				<-start
+				revs[i], errs[i] = managers[i].ClaimEBSProvider(context.Background(), nodes[i], providers[i])
+			})
+		}
+		close(start)
+		wg.Wait()
+
+		rec, rev := recordedEBSProvider(t, js)
+		winner := slices.Index(providers, rec.Provider)
+		require.GreaterOrEqual(t, winner, 0, "recorded provider %q is neither claimant's", rec.Provider)
+		assert.Equal(t, nodes[winner], rec.RecordedBy)
+		require.NoError(t, errs[winner])
+		assert.Equal(t, revs[winner], rev, "the key must still be the winner's Create")
+
+		var mismatch *EBSProviderMismatchError
+		require.ErrorAs(t, errs[1-winner], &mismatch, "loser must be refused")
+		assert.Equal(t, rec.Provider, mismatch.Recorded.Provider)
+	}
+}
+
+// A permanent refusal returns on the first attempt rather than being retried
+// for the startup budget.
+func TestInitServiceWithRetry_PermanentErrorNotRetried(t *testing.T) {
+	stubInitRetrySleep(t)
+	calls := 0
+	_, err := initServiceWithRetry("svc", func() (int, error) {
+		calls++
+		if calls > 1 {
+			t.Fatal("a permanent error was retried")
+		}
+		return 0, fmt.Errorf("refused: %w", errStartupPermanent)
+	})
+	require.ErrorIs(t, err, errStartupPermanent)
+	assert.Equal(t, 1, calls)
+}
+
+// An unset provider resolves to viperblockd for the check, and the daemon
+// refuses on the first mismatch: a retry would need a sleep first.
+func TestDaemonClaimEBSProvider_MismatchRefusesWithoutRetry(t *testing.T) {
+	ns, _, _ := testutil.StartTestJetStream(t)
+	prev := initRetrySleep
+	initRetrySleep = func(time.Duration) { t.Fatal("a mismatch was retried") }
+	t.Cleanup(func() { initRetrySleep = prev })
+
+	first := &Daemon{node: "node1", config: &config.Config{EBS: config.EBSConfig{Provider: config.EBSProviderQEMUNBD}}, jsManager: newLockManager(t, ns.ClientURL())}
+	require.NoError(t, first.claimEBSProvider())
+
+	second := &Daemon{node: "node2", config: &config.Config{}, jsManager: newLockManager(t, ns.ClientURL())}
+	err := second.claimEBSProvider()
+	var mismatch *EBSProviderMismatchError
+	require.ErrorAs(t, err, &mismatch)
+	assert.Equal(t, config.EBSProviderViperblockd, mismatch.Configured)
+}
+
+// A transient KV failure is retried until the bucket answers.
+func TestDaemonClaimEBSProvider_RetriesUntilBucketOpens(t *testing.T) {
+	ns, _, js := testutil.StartTestJetStream(t)
+	m := newLockManager(t, ns.ClientURL())
+	bucket := m.clusterConfig
+	m.clusterConfig = nil
+
+	sleeps := 0
+	prev := initRetrySleep
+	initRetrySleep = func(time.Duration) {
+		sleeps++
+		m.clusterConfig = bucket
+	}
+	t.Cleanup(func() { initRetrySleep = prev })
+
+	d := &Daemon{node: "node1", config: &config.Config{}, jsManager: m}
+	require.NoError(t, d.claimEBSProvider())
+	assert.Equal(t, 1, sleeps)
+	rec, _ := recordedEBSProvider(t, js)
+	assert.Equal(t, config.EBSProviderViperblockd, rec.Provider)
+}
+
+// A daemon whose provider differs from the recorded one exits 1 before any
+// cluster service is built or any instance restored.
+func TestStartCluster_EBSProviderMismatchExits(t *testing.T) {
+	t.Setenv("SPINIFEX_REQUIRE_NATS", "")
+	ns, _, js := testutil.StartTestJetStream(t)
+	seed := newLockManager(t, ns.ClientURL())
+	_, err := seed.ClaimEBSProvider(t.Context(), "node-2", config.EBSProviderQEMUNBD)
+	require.NoError(t, err)
+
+	tmpDir := t.TempDir()
+	cfg := &config.ClusterConfig{
+		Node: "node-1",
+		Nodes: map[string]config.Config{
+			"node-1": {BaseDir: tmpDir, DataDir: tmpDir, NATS: config.NATSConfig{Host: ns.ClientURL()}},
+		},
+	}
+	d, err := NewDaemon(cfg)
+	require.NoError(t, err)
+	t.Cleanup(d.cancel)
+	exitCh := make(chan int, 1)
+	d.exitFunc = func(code int) { exitCh <- code }
+
+	err = d.startCluster()
+	var mismatch *EBSProviderMismatchError
+	require.ErrorAs(t, err, &mismatch)
+	assert.Equal(t, config.EBSProviderViperblockd, mismatch.Configured)
+	select {
+	case code := <-exitCh:
+		assert.Equal(t, 1, code)
+	default:
+		t.Fatal("a provider mismatch must request exit(1)")
+	}
+
+	assert.Nil(t, d.instanceService, "no cluster service may be built after the refusal")
+	assert.Nil(t, d.imageService)
+	assert.Nil(t, d.snapshotService)
+	assert.Nil(t, d.volumeService)
+	assert.Nil(t, d.vpcService)
+	assert.Nil(t, d.eksService)
+	assert.Zero(t, d.vmMgr.Count(), "no instance may be restored after the refusal")
+
+	rec, _ := recordedEBSProvider(t, js)
+	assert.Equal(t, config.EBSProviderQEMUNBD, rec.Provider)
+}

spinifex/runtime/daemon/jetstream.go

@@ -25,6 +25,9 @@ const (
 	InstanceStateBucket = "spinifex-instance-state"
 	// ClusterStateBucket is the name of the KV bucket for cluster state (heartbeats, shutdown markers, service maps).
 	ClusterStateBucket = clusterv1.ClusterStateBucket
+	// ClusterConfigBucket holds settings fixed when a cluster forms. Unlike
+	// ClusterStateBucket it has no TTL, so a key written once stays written.
+	ClusterConfigBucket = "spinifex-cluster-config"
 	// InstanceStatePrefix is the key prefix for the per-node running-set blobs
 	// the record space replaces. Nothing writes it after the cutover; the
 	// migration reads it and a rolled-back node still finds what it left.
@@ -53,6 +56,7 @@ const (
 	// on a bucket stamped 5 rather than reading the keys it no longer owns.
 	InstanceStateBucketVersion      = 5
 	ClusterStateBucketVersion       = 1
+	ClusterConfigBucketVersion      = 1
 	TerminatedInstanceBucketVersion = 3
 )
 
@@ -80,11 +84,12 @@ type JetStreamManager struct {
 	// The per-resource key space the three views above are moving onto. Each
 	// is a third view over a bucket one of them already holds, so the two
 	// spaces share a handle and a recovery driven through either repairs both.
-	records     *kvstore.Store[vm.InstanceRecord] // i.<id>, instance-state bucket
-	termRecords *kvstore.Store[vm.InstanceRecord] // i.<id>, terminated bucket
-	clusterKV   jetstream.KeyValue                // spinifex-cluster-state
-	obs         KVSyncObserver
-	running     runningSetState
+	records       *kvstore.Store[vm.InstanceRecord] // i.<id>, instance-state bucket
+	termRecords   *kvstore.Store[vm.InstanceRecord] // i.<id>, terminated bucket
+	clusterKV     jetstream.KeyValue                // spinifex-cluster-state
+	clusterConfig *kvstore.Bucket                   // spinifex-cluster-config
+	obs           KVSyncObserver
+	running       runningSetState
 }
 
 // checkNodeStateVersion reports whether a node record read from KV is one this
@@ -204,6 +209,29 @@ func (m *JetStreamManager) InitClusterStateBucket() error {
 	return nil
 }
 
+// clusterConfigConfig describes the cluster-config bucket. RecreateIfMissing
+// stays false: a lost lock is not reconstructible by recovery, only by the
+// next daemon start, which claims it again.
+func clusterConfigConfig() kvstore.Config {
+	return kvstore.Config{
+		Name:        ClusterConfigBucket,
+		Description: "Spinifex cluster configuration fixed at formation (EBS provider)",
+		History:     1,
+		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
+			return migrate.DefaultRegistry.RunKV(ctx, ClusterConfigBucket, kv, ClusterConfigBucketVersion)
+		},
+		Missing: "cluster config KV bucket not initialized",
+	}
+}
+
+// InitClusterConfigBucket initializes the cluster-config KV bucket, creating it
+// if it doesn't exist.
+func (m *JetStreamManager) InitClusterConfigBucket() error {
+	m.clusterConfig = kvstore.NewBucket(m.js, clusterConfigConfig())
+	_, err := m.clusterConfig.KV(context.Background())
+	return err
+}
+
 // InitTerminatedInstanceBucket initializes the terminated-instances KV bucket with a 1-hour TTL.
 // JetStream automatically purges keys after 1 hour, matching AWS behavior for terminated instances.
 func (m *JetStreamManager) InitTerminatedInstanceBucket() error {

spinifex/runtime/roles/awsgw/awsgw.go

@@ -725,6 +725,9 @@ func launchService(config *config.ClusterConfig) error {
 		SQSWorkerPolling:       sqsRelay.WorkerPresent,
 		SQSTenantAPIRegistered: func() bool { return sqsServed(composedServices) },
 	})
+	operatorAdminDeps = withPlatformCleaner(operatorAdminDeps, func() (accountlifecycle.PlatformCleaner, error) {
+		return platformCleaner(janitorCtx, &nodeConfig, natsConn, js, operatorAdminDeps.NodeIDs)
+	})
 	operatorControlPlane := operatoradmin.NewRegistration(operatorAdminDeps)
 	// Every gateway scans for account deletions whose holder died; the fenced
 	// takeover makes any number of them safe.

spinifex/runtime/roles/awsgw/compose_test.go

@@ -1,12 +1,16 @@
 package awsgw
 
 import (
+	"context"
+	"errors"
 	"go/ast"
 	"go/parser"
 	"go/token"
 	"testing"
 
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	"github.com/mulgadc/spinifex/spinifex/operator/accountlifecycle"
 	operatoradmin "github.com/mulgadc/spinifex/spinifex/operator/admin"
 
 	awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
@@ -88,3 +92,40 @@ func TestAwsgwStartsTheDeletionResumer(t *testing.T) {
 	})
 	assert.True(t, started, "awsgw.go must call Deps.StartDeletionResumer")
 }
+
+type fixedCleaner struct{}
+
+func (fixedCleaner) CleanupPlatform(context.Context, *accountcleanup.Request) (*accountcleanup.Result, error) {
+	return &accountcleanup.Result{}, nil
+}
+
+// A non-finalizing DeleteAccount cleans through the in-process owners awsgw
+// binds. A cleanup that cannot be built must leave the field nil, which refuses
+// the request, rather than bind something half-working.
+func TestWithPlatformCleaner_BindsTheCleanupOrLeavesItNil(t *testing.T) {
+	bound := withPlatformCleaner(operatoradmin.Deps{}, func() (accountlifecycle.PlatformCleaner, error) { return fixedCleaner{}, nil })
+	assert.NotNil(t, bound.PlatformCleaner)
+
+	unbound := withPlatformCleaner(operatoradmin.Deps{}, func() (accountlifecycle.PlatformCleaner, error) {
+		return nil, errors.New("owner store unavailable")
+	})
+	assert.Nil(t, unbound.PlatformCleaner)
+}
+
+// The cleanup is bound where the operator deps are composed, or the exercise is
+// refused on every gateway.
+func TestAwsgwBindsThePlatformCleanup(t *testing.T) {
+	file, err := parser.ParseFile(token.NewFileSet(), "awsgw.go", nil, 0)
+	require.NoError(t, err)
+
+	var bound bool
+	ast.Inspect(file, func(n ast.Node) bool {
+		if call, ok := n.(*ast.CallExpr); ok {
+			if ident, ok := call.Fun.(*ast.Ident); ok && ident.Name == "withPlatformCleaner" {
+				bound = true
+			}
+		}
+		return true
+	})
+	assert.True(t, bound, "awsgw.go must call withPlatformCleaner")
+}

spinifex/runtime/roles/awsgw/platform_cleanup.go

@@ -0,0 +1,66 @@
+package awsgw
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	ec2key "github.com/mulgadc/spinifex/spinifex/domains/ec2/key"
+	ec2launchtemplate "github.com/mulgadc/spinifex/spinifex/domains/ec2/launchtemplate"
+	ec2placementgroup "github.com/mulgadc/spinifex/spinifex/domains/ec2/placementgroup"
+	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+	ec2tags "github.com/mulgadc/spinifex/spinifex/domains/ec2/tags"
+	"github.com/mulgadc/spinifex/spinifex/operator/accountlifecycle"
+	operatoradmin "github.com/mulgadc/spinifex/spinifex/operator/admin"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// platformCleaner builds the EC2 platform cleanup over the owners' own services
+// in this process. It has no NATS subject, so the only caller is the operator
+// admin DeleteAccount that holds it. Deleting a key pair, placement group or
+// launch template through these owners clears its tag-index entry too.
+func platformCleaner(ctx context.Context, node *config.Config, nc *nats.Conn, js jetstream.JetStream, nodeIDs []string) (accountlifecycle.PlatformCleaner, error) {
+	tagsKV, err := ec2tags.GetOrCreateTagsBucket(ctx, js)
+	if err != nil {
+		return nil, fmt.Errorf("tags bucket: %w", err)
+	}
+	tags := ec2tags.NewTagsServiceImpl(node, tagsKV)
+	keys := ec2key.NewKeyServiceImpl(node)
+	groups, err := ec2placementgroup.NewPlacementGroupServiceImplWithNATS(ctx, node, nc)
+	if err != nil {
+		return nil, fmt.Errorf("placement group service: %w", err)
+	}
+	templates, err := ec2launchtemplate.NewLaunchTemplateServiceImplWithNATS(ctx, node, nc)
+	if err != nil {
+		return nil, fmt.Errorf("launch template service: %w", err)
+	}
+	spot, err := ec2spotinstance.NewSpotInstanceServiceImplWithNATS(ctx, node, nc)
+	if err != nil {
+		return nil, fmt.Errorf("spot instance service: %w", err)
+	}
+	keys.SetCentralTagStore(tags)
+	groups.SetCentralTagStore(tags)
+	templates.SetCentralTagStore(tags)
+
+	return accountcleanup.New(keys, groups, templates, tags, accountcleanup.ReferenceReaders{
+		Instances: accountlifecycle.NewInstanceReferenceReader(nc, nodeIDs),
+		Spot:      spot,
+		Templates: templates,
+	}), nil
+}
+
+// withPlatformCleaner binds the cleanup a non-finalizing DeleteAccount runs. When
+// it cannot be built the field stays nil, which refuses that request instead of
+// half-running it.
+func withPlatformCleaner(d operatoradmin.Deps, build func() (accountlifecycle.PlatformCleaner, error)) operatoradmin.Deps {
+	cleaner, err := build()
+	if err != nil {
+		slog.Error("awsgw: EC2 platform cleanup unavailable", "err", err)
+		return d
+	}
+	d.PlatformCleaner = cleaner
+	return d
+}

spinifex/runtime/roles/qemunbdd/conformance_test.go

@@ -7,6 +7,8 @@ package qemunbdd_test
 import (
 	"os"
 	"os/exec"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -14,6 +16,8 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs/conformance"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs/natsserve"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/qemunbdd"
 	"github.com/stretchr/testify/require"
 )
@@ -58,9 +62,23 @@ func newQEMUProviderTB(t testing.TB) ebsprovider.EBSProvider {
 	return provider
 }
 
+// killQEMUExport stops the qemu-nbd serving an export behind the provider's
+// back, through the pid file it writes beside the socket, as a crash would.
+func killQEMUExport(t *testing.T, published *ebsprovider.PublishedVolume) {
+	t.Helper()
+	serverType, socketPath, _, _, err := nbd.ParseNBDURI(published.Export.NBD.NBDURI)
+	require.NoError(t, err)
+	require.Equal(t, "unix", serverType)
+	data, err := os.ReadFile(strings.TrimSuffix(socketPath, ".sock") + ".pid")
+	require.NoError(t, err)
+	pid, err := strconv.Atoi(strings.TrimSpace(string(data)))
+	require.NoError(t, err)
+	require.NoError(t, hostprocess.ForceKillProcess(pid, 5*time.Second))
+}
+
 func TestQEMUNBDProviderConformance(t *testing.T) {
 	requireQEMUTools(t)
-	conformance.RunSuite(t, newQEMUProvider)
+	conformance.RunSuiteWithConfig(t, newQEMUProvider, conformance.SuiteConfig{KillExport: killQEMUExport})
 }
 
 // TestQEMUNBDExternalNBDClient checks the half of the boundary our own Go
@@ -76,12 +94,12 @@ func TestQEMUNBDExternalNBDClient(t *testing.T) {
 // in it, and a client that cannot tell the difference.
 func TestQEMUNBDProviderNATSConformance(t *testing.T) {
 	requireQEMUTools(t)
-	conformance.RunSuite(t, func(t *testing.T) ebsprovider.EBSProvider {
+	conformance.RunSuiteWithConfig(t, func(t *testing.T) ebsprovider.EBSProvider {
 		t.Helper()
 		_, conn := testutil.StartTestNATS(t)
 		stop, err := natsserve.Serve(t.Context(), conn, newQEMUProvider(t), natsserve.Options{})
 		require.NoError(t, err)
 		t.Cleanup(stop)
 		return ebsprovider.NewNATSProvider(conn, 15*time.Second)
-	})
+	}, conformance.SuiteConfig{KillExport: killQEMUExport})
 }

spinifex/runtime/roles/qemunbdd/live_lifecycle_test.go

@@ -76,13 +76,15 @@ func TestLive_QEMUNBDLifecycle(t *testing.T) {
 		cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 		defer cancel()
 		cleanupBestEffort(t, "unpublish restored volume", func() error {
-			return provider.UnpublishVolume(cleanupCtx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: restoredVolumeID, NodeID: nodeID})
+			_, err := provider.UnpublishVolume(cleanupCtx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: restoredVolumeID, NodeID: nodeID})
+			return err
 		})
 		cleanupBestEffort(t, "delete restored volume", func() error {
 			return provider.DeleteVolume(cleanupCtx, ebsprovider.DeleteVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: restoredVolumeID})
 		})
 		cleanupBestEffort(t, "unpublish volume", func() error {
-			return provider.UnpublishVolume(cleanupCtx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: nodeID})
+			_, err := provider.UnpublishVolume(cleanupCtx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: nodeID})
+			return err
 		})
 		cleanupBestEffort(t, "delete snapshot", func() error {
 			return provider.DeleteSnapshot(cleanupCtx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID})
@@ -110,7 +112,8 @@ func TestLive_QEMUNBDLifecycle(t *testing.T) {
 	writePattern(t, sockPath, pattern, liveVolumeBytes)
 	readAndVerifyPattern(t, sockPath, pattern, liveVolumeBytes)
 
-	require.NoError(t, provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: nodeID}), "unpublish volume")
+	_, err = provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: nodeID})
+	require.NoError(t, err, "unpublish volume")
 	t.Logf("unpublished volume %s", volumeID)
 
 	snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID})
@@ -136,7 +139,8 @@ func TestLive_QEMUNBDLifecycle(t *testing.T) {
 	// snapshot and the restore into an independent volume.
 	readAndVerifyPattern(t, restoredSockPath, pattern, liveVolumeBytes)
 
-	require.NoError(t, provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: restoredVolumeID, NodeID: nodeID}), "unpublish restored volume")
+	_, err = provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: restoredVolumeID, NodeID: nodeID})
+	require.NoError(t, err, "unpublish restored volume")
 	t.Logf("unpublished restored volume %s", restoredVolumeID)
 }
 

spinifex/runtime/roles/qemunbdd/provider.go

@@ -14,9 +14,11 @@ import (
 	"strings"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 )
 
 // nbdSharedClients bounds how many simultaneous NBD connections qemu-nbd
@@ -69,8 +71,27 @@ type publication struct {
 	nodeID     string
 	socketPath string
 	pid        int
+	readOnly   bool
 }
 
+// dead reports positive evidence that the export is no longer serving: its
+// qemu-nbd has exited or its socket is gone. Without a PID nothing is known.
+// A qemu-nbd whose socket is gone can still be writing for a connected client.
+func (p *publication) dead() bool {
+	if p.pid <= 0 {
+		return false
+	}
+	if !hostprocess.ProcessAlive(p.pid) {
+		return true
+	}
+	_, err := os.Stat(p.socketPath)
+	return errors.Is(err, os.ErrNotExist)
+}
+
+// defaultStopWait bounds the wait for a qemu-nbd to exit after SIGTERM
+// before a republish over its dead socket is refused.
+const defaultStopWait = 10 * time.Second
+
 // Provider is an EBSProvider backed by qcow2 files and qemu-nbd. One volume
 // is one qcow2 file; a snapshot is an independent copy, not a chunk
 // reference, so DeleteVolume never has to consult a snapshot the way viperblock does.
@@ -78,6 +99,9 @@ type Provider struct {
 	baseDir string
 	run     runner
 
+	// stopWait overrides defaultStopWait. Zero means the default.
+	stopWait time.Duration
+
 	mu           sync.Mutex
 	published    map[string]*publication
 	volumeMeta   map[string]volumeMeta
@@ -131,11 +155,11 @@ func (p *Provider) pidFilePath(volumeID string) string {
 }
 
 // checkVersion mirrors ebsprovider's supported schema window. qemunbdd's
-// normal operations have the same representation in both versions; the NATS
+// normal operations have the same representation in every version; the NATS
 // server owns the version-specific PublishVolume wire conversion.
 func checkVersion(version uint16) error {
 	if !ebsprovider.SupportsSchemaVersion(version) {
-		return fmt.Errorf("%w: got %d, supported versions are %d and %d", ebsprovider.ErrUnsupportedVersion, version, ebsprovider.LegacySchemaVersion, ebsprovider.SchemaVersion)
+		return fmt.Errorf("%w: got %d, %s", ebsprovider.ErrUnsupportedVersion, version, ebsprovider.SupportedVersionsText())
 	}
 	return nil
 }
@@ -148,7 +172,11 @@ func (p *Provider) GetCapabilities(_ context.Context, req ebsprovider.GetCapabil
 	if err := checkVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
-	return &ebsprovider.GetCapabilitiesResponse{Versioned: ebsprovider.NewVersioned(), Capabilities: capabilities}, nil
+	return &ebsprovider.GetCapabilitiesResponse{
+		Versioned:        ebsprovider.ReplyVersioned(req.SchemaVersion),
+		Capabilities:     capabilities,
+		ContractVersions: ebsprovider.SupportedSchemaVersions(),
+	}, nil
 }
 
 // qemuImgInfo is the subset of `qemu-img info --output=json` this provider
@@ -350,7 +378,7 @@ func (p *Provider) ListVolumes(_ context.Context, req ebsprovider.ListVolumesReq
 		return nil, fmt.Errorf("qemunbdd: read volumes directory: %w", err)
 	}
 
-	response := &ebsprovider.ListVolumesResponse{Versioned: ebsprovider.NewVersioned()}
+	response := &ebsprovider.ListVolumesResponse{Versioned: ebsprovider.ReplyVersioned(req.SchemaVersion)}
 	response.Volumes, response.NextToken = ebsprovider.Page(qcow2IDs(entries), req.StartingToken, int(req.PageSize()),
 		func(id string) ebsprovider.VolumeRef {
 			return ebsprovider.VolumeRef{ID: id, Handle: p.volumePath(id)}
@@ -527,7 +555,7 @@ func (p *Provider) ListSnapshots(_ context.Context, req ebsprovider.ListSnapshot
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
-	response := &ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.NewVersioned()}
+	response := &ebsprovider.ListSnapshotsResponse{Versioned: ebsprovider.ReplyVersioned(req.SchemaVersion)}
 	response.Snapshots, response.NextToken = ebsprovider.Page(qcow2IDs(entries), req.StartingToken, int(req.PageSize()),
 		func(id string) ebsprovider.SnapshotRef {
 			return ebsprovider.SnapshotRef{
@@ -686,9 +714,21 @@ func (p *Provider) PublishVolume(ctx context.Context, req ebsprovider.PublishVol
 	}
 	if pub, ok := p.published[req.VolumeID]; ok {
 		if pub.nodeID != req.NodeID {
-			return nil, fmt.Errorf("%w: volume %s is published to %s", ebsprovider.ErrVolumeInUse, req.VolumeID, pub.nodeID)
+			return nil, fmt.Errorf("%w: volume %s is published to %s", ebsprovider.HeldElsewhereError(req.SchemaVersion), req.VolumeID, pub.nodeID)
+		}
+		// A dead export's entry is dropped and the volume exported again, so
+		// a republish after a crash never returns a socket nothing serves.
+		if pub.dead() {
+			if err := p.stopOrphanedExport(pub); err != nil {
+				return nil, fmt.Errorf("%w: volume %s: %w", ebsprovider.ErrUnavailable, req.VolumeID, err)
+			}
+			delete(p.published, req.VolumeID)
+		} else {
+			if pub.readOnly != req.ReadOnly {
+				return nil, fmt.Errorf("%w: volume %s is already published read_only=%t", ebsprovider.ErrVolumeInUse, req.VolumeID, pub.readOnly)
+			}
+			return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: pub.nodeID, Export: ebsprovider.NewNBDExport(nbd.FormatNBDSocketURI(pub.socketPath))}, nil
 		}
-		return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: pub.nodeID, Export: ebsprovider.NewNBDExport(nbd.FormatNBDSocketURI(pub.socketPath))}, nil
 	}
 
 	sockPath := p.socketPath(req.VolumeID)
@@ -717,10 +757,27 @@ func (p *Provider) PublishVolume(ctx context.Context, req ebsprovider.PublishVol
 	}
 
 	pid := readPID(pidPath)
-	p.published[req.VolumeID] = &publication{nodeID: req.NodeID, socketPath: sockPath, pid: pid}
+	p.published[req.VolumeID] = &publication{nodeID: req.NodeID, socketPath: sockPath, pid: pid, readOnly: req.ReadOnly}
 	return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: ebsprovider.NewNBDExport(nbd.FormatNBDSocketURI(sockPath))}, nil
 }
 
+// stopOrphanedExport stops a dead entry's qemu-nbd if it is still running,
+// since a connected client keeps writing through it and a second export would
+// make two writers. It stops it as UnpublishVolume does, with SIGTERM.
+func (p *Provider) stopOrphanedExport(pub *publication) error {
+	if !hostprocess.ProcessAlive(pub.pid) {
+		return nil
+	}
+	wait := p.stopWait
+	if wait == 0 {
+		wait = defaultStopWait
+	}
+	if !hostprocess.TerminateProcess(pub.pid, wait) {
+		return fmt.Errorf("qemu-nbd %d lost its socket and did not exit", pub.pid)
+	}
+	return nil
+}
+
 // readPID best-effort reads the PID qemu-nbd's --fork wrote to pidPath. A
 // missing or unparsable file (as in tests, where no real qemu-nbd runs)
 // yields 0, meaning UnpublishVolume will skip signaling a process.
@@ -736,20 +793,23 @@ func readPID(pidPath string) int {
 	return pid
 }
 
-func (p *Provider) UnpublishVolume(_ context.Context, req ebsprovider.UnpublishVolumeRequest) error {
-	if err := checkVersion(req.SchemaVersion); err != nil {
-		return err
+// UnpublishVolume honours NoSeal trivially: qemu-nbd writes straight to the
+// volume's qcow2 file, so there is no separate copy to seal or to drop.
+func (p *Provider) UnpublishVolume(_ context.Context, req ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
+	if err := ebsprovider.CheckUnpublishVersion(req); err != nil {
+		return nil, err
 	}
 	if req.VolumeID == "" || req.NodeID == "" {
-		return fmt.Errorf("%w: volume and node IDs are required", ebsprovider.ErrInvalidArgument)
+		return nil, fmt.Errorf("%w: volume and node IDs are required", ebsprovider.ErrInvalidArgument)
 	}
 
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
+	result := &ebsprovider.UnpublishedVolume{}
 	pub, ok := p.published[req.VolumeID]
 	if !ok || pub.nodeID != req.NodeID {
-		return nil
+		return result, nil
 	}
 	if pub.pid > 0 {
 		if proc, err := os.FindProcess(pub.pid); err == nil {
@@ -759,5 +819,18 @@ func (p *Provider) UnpublishVolume(_ context.Context, req ebsprovider.UnpublishV
 	_ = os.Remove(pub.socketPath)
 	_ = os.Remove(p.pidFilePath(req.VolumeID))
 	delete(p.published, req.VolumeID)
-	return nil
+	result.TornDown = req.SchemaVersion >= ebsprovider.NoSealSchemaVersion
+	return result, nil
+}
+
+// HoldsUnsealedWrites answers that none are held: the qcow2 file qemu-nbd
+// writes is the backend, so no write is held anywhere it does not already have.
+func (p *Provider) HoldsUnsealedWrites(_ context.Context, nodeID string, volumeIDs []string) ([]string, error) {
+	if nodeID == "" {
+		return nil, fmt.Errorf("%w: node ID is required", ebsprovider.ErrInvalidArgument)
+	}
+	if slices.Contains(volumeIDs, "") {
+		return nil, fmt.Errorf("%w: empty volume ID", ebsprovider.ErrInvalidArgument)
+	}
+	return nil, nil
 }

spinifex/runtime/roles/qemunbdd/provider_test.go

@@ -3,13 +3,17 @@ package qemunbdd
 import (
 	"context"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
 )
 
 // call records one invocation the fake runner captured, so tests can assert
@@ -621,14 +625,14 @@ func TestUnpublishVolume_Success(t *testing.T) {
 	_, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub", NodeID: "node-1"})
 	require.NoError(t, err)
 
-	require.NoError(t, p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub", NodeID: "node-1"}))
+	require.NoError(t, unpublishErr(p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub", NodeID: "node-1"})))
 	_, err = os.Stat(p.socketPath("vol-unpub"))
 	require.True(t, os.IsNotExist(err))
 }
 
 func TestUnpublishVolume_AbsentIsIdempotent(t *testing.T) {
 	p, _ := newTestProvider(t)
-	require.NoError(t, p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-never-existed", NodeID: "node-1"}))
+	require.NoError(t, unpublishErr(p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-never-existed", NodeID: "node-1"})))
 }
 
 func TestUnpublishVolume_RepeatIsIdempotent(t *testing.T) {
@@ -638,8 +642,8 @@ func TestUnpublishVolume_RepeatIsIdempotent(t *testing.T) {
 	_, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub-repeat", NodeID: "node-1"})
 	require.NoError(t, err)
 
-	require.NoError(t, p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub-repeat", NodeID: "node-1"}))
-	require.NoError(t, p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub-repeat", NodeID: "node-1"}))
+	require.NoError(t, unpublishErr(p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub-repeat", NodeID: "node-1"})))
+	require.NoError(t, unpublishErr(p.UnpublishVolume(context.Background(), ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub-repeat", NodeID: "node-1"})))
 
 	// Republishing after unpublish must start a fresh qemu-nbd.
 	_, err = p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-unpub-repeat", NodeID: "node-1"})
@@ -666,3 +670,79 @@ func TestGetCapabilities_Fixed(t *testing.T) {
 	assert.False(t, resp.Capabilities.OnlineExpansion)
 	assert.False(t, resp.Capabilities.OwnerRouting)
 }
+
+func unpublishErr(_ *ebsprovider.UnpublishedVolume, err error) error { return err }
+
+// runningExport starts a process standing in for a qemu-nbd that lost its
+// socket but kept running, reaped on its own goroutine as a real fork would be.
+func runningExport(t *testing.T) int {
+	t.Helper()
+	hostprocess.ShortenProcessExitPollForTest(t, 5*time.Millisecond)
+	cmd := exec.Command("sleep", "300")
+	require.NoError(t, cmd.Start())
+	reaped := make(chan struct{})
+	go func() { defer close(reaped); _ = cmd.Wait() }()
+	t.Cleanup(func() { _ = cmd.Process.Kill(); <-reaped })
+	return cmd.Process.Pid
+}
+
+// unstoppableExport returns a zombie: kill(pid,0) still reaches it and no
+// signal makes it exit, so it stands in for a qemu-nbd that will not stop.
+func unstoppableExport(t *testing.T) int {
+	t.Helper()
+	hostprocess.ShortenProcessExitPollForTest(t, 5*time.Millisecond)
+	cmd := exec.Command("sleep", "300")
+	require.NoError(t, cmd.Start())
+	require.NoError(t, cmd.Process.Kill())
+	var info unix.Siginfo
+	require.NoError(t, unix.Waitid(unix.P_PID, cmd.Process.Pid, &info, unix.WEXITED|unix.WNOWAIT, nil))
+	t.Cleanup(func() { _ = cmd.Wait() })
+	return cmd.Process.Pid
+}
+
+// aliveAtRun records, at the moment qemu-nbd is run, whether pid is alive.
+type aliveAtRun struct {
+	*fakeRunner
+
+	pid       int
+	aliveSeen []bool
+}
+
+func (r *aliveAtRun) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
+	if name == "qemu-nbd" {
+		r.aliveSeen = append(r.aliveSeen, hostprocess.ProcessAlive(r.pid))
+	}
+	return r.fakeRunner.Run(ctx, name, args...)
+}
+
+// A qemu-nbd whose socket is gone still serves a client already connected to
+// it, so re-exporting beside it makes two writers on one qcow2.
+func TestPublishVolume_OrphanStoppedFirst(t *testing.T) {
+	p, fr := newTestProvider(t)
+	const volumeID = "vol-orph"
+	require.NoError(t, os.WriteFile(p.volumePath(volumeID), nil, 0o640))
+	pid := runningExport(t)
+	run := &aliveAtRun{fakeRunner: fr, pid: pid}
+	p.run = run
+	p.published[volumeID] = &publication{nodeID: "node-1", socketPath: p.socketPath(volumeID), pid: pid}
+
+	_, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: volumeID, NodeID: "node-1"})
+	require.NoError(t, err)
+	assert.Equal(t, []bool{false}, run.aliveSeen, "the old qemu-nbd must be gone before the volume is exported again")
+}
+
+// A qemu-nbd that will not stop is still a writer, so a second export is
+// refused as retry-later and the entry kept so the next attempt tries again.
+func TestPublishVolume_OrphanUnstoppable(t *testing.T) {
+	p, fr := newTestProvider(t)
+	p.stopWait = 50 * time.Millisecond
+	const volumeID = "vol-orphan-stuck"
+	require.NoError(t, os.WriteFile(p.volumePath(volumeID), nil, 0o640))
+	pub := &publication{nodeID: "node-1", socketPath: p.socketPath(volumeID), pid: unstoppableExport(t)}
+	p.published[volumeID] = pub
+
+	_, err := p.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: volumeID, NodeID: "node-1"})
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable)
+	assert.Same(t, pub, p.published[volumeID], "the entry must stay while its process may still write")
+	assert.Empty(t, fr.callsSnapshot(), "no second qemu-nbd may start beside one that would not stop")
+}

tests/integration/lambda_test.go

@@ -59,11 +59,9 @@ func startLambdaDaemonLite(t *testing.T, nc *nats.Conn) {
 	sub(t, nc, lambdafunction.SubjectRollbackCreate, func(m *nats.Msg) { dispatch(m, svc.RollbackCreate) })
 }
 
-// TestLambda_SignedCreateAndGetDownloadsTheStoredCode drives CreateFunction and
-// GetFunction through signed ingress, the NATS-owned function record and the
-// real code store on the Predastore fixture, then downloads Code.Location.
-// The fixture authenticates only its service key, so that key stands in for
-// the scoped code-store session; provisioning and session scoping are not here.
+// Signed Create and the Get reads over NATS metadata and the Predastore
+// fixture's code store, then downloads Code.Location. The fixture's service
+// key stands in for the scoped code-store session, which is untested.
 func TestLambda_SignedCreateAndGetDownloadsTheStoredCode(t *testing.T) {
 	fixture := testpredastore.Start(t)
 	bucket := codestore.BucketName(testRegion)
@@ -132,6 +130,26 @@ func TestLambda_SignedCreateAndGetDownloadsTheStoredCode(t *testing.T) {
 	require.Equal(t, "e2e", aws.StringValue(got.Configuration.Environment.Variables["MODE"]))
 	require.NotNil(t, got.Code.Location)
 
+	cfg, err := cli.GetFunctionConfiguration(&lambda.GetFunctionConfigurationInput{FunctionName: aws.String("e2e-fn")})
+	require.NoError(t, err)
+	require.Equal(t, got.Configuration.String(), cfg.String())
+
+	signing, err := cli.GetFunctionCodeSigningConfig(&lambda.GetFunctionCodeSigningConfigInput{FunctionName: aws.String("e2e-fn")})
+	require.NoError(t, err)
+	require.Nil(t, signing.CodeSigningConfigArn)
+	require.Nil(t, signing.FunctionName)
+	versions, err := cli.ListVersionsByFunction(&lambda.ListVersionsByFunctionInput{FunctionName: aws.String("e2e-fn"), MaxItems: aws.Int64(10000)})
+	require.NoError(t, err)
+	require.Len(t, versions.Versions, 1)
+	require.Equal(t, aws.StringValue(got.Configuration.FunctionArn)+":$LATEST", aws.StringValue(versions.Versions[0].FunctionArn))
+	require.Equal(t, "$LATEST", aws.StringValue(versions.Versions[0].Version))
+	require.Nil(t, versions.NextMarker)
+
+	_, err = cli.GetFunctionCodeSigningConfig(&lambda.GetFunctionCodeSigningConfigInput{FunctionName: aws.String("e2e-missing")})
+	var aerr awserr.Error
+	require.ErrorAs(t, err, &aerr)
+	require.Equal(t, lambda.ErrCodeResourceNotFoundException, aerr.Code())
+
 	// SSL_CERT_FILE (set by the fixture) makes the default client trust it.
 	resp, err := http.Get(aws.StringValue(got.Code.Location)) //nolint:gosec,noctx // URL is the presigned location under test
 	require.NoError(t, err)