feat(uefi): add the instance UEFI variable store pointer, store records and their bucket (#1499)

InstanceStatus and VM carry an optional UEFIVarStoreRef naming the store
object and the firmware it was written under. The new uefistore package
holds FirmwareIdentity, UEFIStoreRecord and UEFIWriteBack, and a Store over
the spinifex-uefi-stores bucket (history 1, no TTL, schema 1) whose writes
are all revision-guarded and validated. The daemon creates the bucket at
startup. Nothing reads or writes either yet.

contracts/ec2/v1/README.md

@@ -23,7 +23,7 @@ subject shape are wire compatibility. Callers and receivers must use this
 package rather than rebuild those details. `commands_test.go` preserves the
 version-1 JSON shape and subject form.
 
-This package owns shared command payloads and the drain-volume response. It
+This package owns shared command payloads. It
 does not define every action's reply body, authorize the request, store EC2
 state or implement NATS transport. Those remain responsibilities of the EC2
 domain and its runtime composition.

contracts/ec2/v1/commands.go

@@ -29,7 +29,6 @@ type EC2InstanceCommand struct {
 	Attributes                EC2CommandAttributes       `json:"attributes"`
 	AttachVolumeData          *AttachVolumeData          `json:"attach_volume_data,omitempty"`
 	DetachVolumeData          *DetachVolumeData          `json:"detach_volume_data,omitempty"`
-	DrainVolumeData           *DrainVolumeData           `json:"drain_volume_data,omitempty"`
 	AttachENIData             *AttachENIData             `json:"attach_eni_data,omitempty"`
 	DetachENIData             *DetachENIData             `json:"detach_eni_data,omitempty"`
 	IamProfileAssociationData *IamProfileAssociationData `json:"iam_profile_association_data,omitempty"`
@@ -45,7 +44,6 @@ type EC2CommandAttributes struct {
 	StartInstance               bool `json:"start_instance"`
 	AttachVolume                bool `json:"attach_volume"`
 	DetachVolume                bool `json:"detach_volume"`
-	DrainVolume                 bool `json:"drain_volume,omitempty"`
 	RebootInstance              bool `json:"reboot_instance"`
 	AttachENI                   bool `json:"attach_eni"`
 	DetachENI                   bool `json:"detach_eni"`
@@ -69,33 +67,6 @@ type DetachVolumeData struct {
 	Force    bool   `json:"force,omitempty"`
 }
 
-// DrainVolumeData carries parameters for a drain-volume command, which flushes
-// the volume's in-flight writes to S3 before a snapshot reads them. The command
-// is addressed to the instance the volume is attached to because the drain
-// socket only exists on the node hosting it.
-type DrainVolumeData struct {
-	VolumeID string `json:"volume_id"`
-}
-
-const (
-	// DrainVolumeStatusDrained is the ack a node returns once the volume's
-	// in-flight writes have reached S3.
-	DrainVolumeStatusDrained = "drained"
-
-	// DrainVolumeStatusNotRunning is the ack a node returns when it still holds
-	// the instance but the VM is not running, so no process is writing to the
-	// volume and its sealed checkpoint is already current. An attachment record
-	// outlives the writer — stop deliberately leaves boot volumes attached — so
-	// the caller cannot tell this from the record alone.
-	DrainVolumeStatusNotRunning = "not-running"
-)
-
-// DrainVolumeResponse is the reply to a drain-volume command.
-type DrainVolumeResponse struct {
-	VolumeID string `json:"volume_id"`
-	Status   string `json:"status"`
-}
-
 // AttachENIData carries parameters for an attach-network-interface command.
 type AttachENIData struct {
 	NetworkInterfaceID string `json:"network_interface_id"`

contracts/ec2/v1/commands_test.go

@@ -16,7 +16,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			StartInstance:               true,
 			AttachVolume:                true,
 			DetachVolume:                true,
-			DrainVolume:                 true,
 			RebootInstance:              true,
 			AttachENI:                   true,
 			DetachENI:                   true,
@@ -28,7 +27,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		},
 		AttachVolumeData:          &AttachVolumeData{VolumeID: "vol-1", Device: "/dev/sdf"},
 		DetachVolumeData:          &DetachVolumeData{VolumeID: "vol-2", Device: "/dev/sdg", Force: true},
-		DrainVolumeData:           &DrainVolumeData{VolumeID: "vol-3"},
 		AttachENIData:             &AttachENIData{NetworkInterfaceID: "eni-1", DeviceIndex: 2},
 		DetachENIData:             &DetachENIData{AttachmentID: "eni-attach-1", Force: true},
 		IamProfileAssociationData: &IamProfileAssociationData{InstanceProfileArn: "arn:aws:iam::123456789012:instance-profile/example"},
@@ -47,7 +45,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			"start_instance": true,
 			"attach_volume": true,
 			"detach_volume": true,
-			"drain_volume": true,
 			"reboot_instance": true,
 			"attach_eni": true,
 			"detach_eni": true,
@@ -59,7 +56,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		},
 		"attach_volume_data": {"volume_id": "vol-1", "device": "/dev/sdf"},
 		"detach_volume_data": {"volume_id": "vol-2", "device": "/dev/sdg", "force": true},
-		"drain_volume_data": {"volume_id": "vol-3"},
 		"attach_eni_data": {"network_interface_id": "eni-1", "device_index": 2},
 		"detach_eni_data": {"attachment_id": "eni-attach-1", "force": true},
 		"iam_profile_association_data": {"instance_profile_arn": "arn:aws:iam::123456789012:instance-profile/example"},
@@ -69,16 +65,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 	}`, string(data))
 }
 
-func TestDrainVolumeResponseJSONContract(t *testing.T) {
-	data, err := json.Marshal(DrainVolumeResponse{
-		VolumeID: "vol-1",
-		Status:   DrainVolumeStatusDrained,
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"volume_id":"vol-1","status":"drained"}`, string(data))
-	require.Equal(t, "not-running", DrainVolumeStatusNotRunning)
-}
-
 func TestInstanceCommandSubjects(t *testing.T) {
 	require.Equal(t, "ec2.cmd.i-0123456789abcdef0", InstanceCommandSubject("i-0123456789abcdef0"))
 	require.Equal(t, "ec2.cmd.*", InstanceCommandSubjectWildcard)

contracts/viperblockd/legacy/v1/README.md

@@ -51,19 +51,6 @@ request/reply replies.
 removed its local state and export. These routes and JSON shapes are deployed
 legacy compatibility behaviour, not a generic EBS-provider interface.
 
-## Fence route
-
-`VolumeFencedSubject(node)` is `ebs.<node>.fenced`, or `ebs.fenced` with no
-node for a single-node daemon. The Viperblock adapter publishes
-`VolumeFencedEvent` on it after tearing down an export whose volume lease
-moved to another node; the local daemon subscribes to stop the guest that
-was using it. Node-addressed, like mount and unmount: the guest is local, and
-no other node's daemon has anything to stop.
-
-The event is sent as a request and repeated until the daemon replies with `VolumeFencedAck`, because the fence often runs while the node's NATS server is restarting and a single publish can reach no subscriber.
-The daemon replies only when the event carries a reply subject, so a plain publish is still accepted.
-Repeats are harmless, since fencing a guest that is already fenced changes nothing.
-
 ## Abandon route
 
 `VolumeAbandonSubject(node)` is `ebs.<node>.abandon`. The daemon uses

contracts/viperblockd/legacy/v1/abandon_test.go

@@ -9,9 +9,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// TestVolumeAbandonSubject_BothForms pins the abandon route the same way the
-// fenced route is pinned: node-addressed, with an empty-node fallback for a
-// single-node daemon.
+// TestVolumeAbandonSubject_BothForms pins the abandon route: node-addressed,
+// with an empty-node fallback for a single-node daemon.
 func TestVolumeAbandonSubject_BothForms(t *testing.T) {
 	assert.Equal(t, "ebs.node-a.abandon", viperblocklegacyv1.VolumeAbandonSubject("node-a"))
 	assert.Equal(t, "ebs.abandon", viperblocklegacyv1.VolumeAbandonSubject(""))

contracts/viperblockd/legacy/v1/fence.go

@@ -1,29 +0,0 @@
-package viperblocklegacyv1
-
-// VolumeFencedSubject is where a fenced volume is announced so the daemon on
-// this node can stop the guest that was using it. Node-addressed: the guest is
-// local, and every other node's daemon has nothing to do about it.
-func VolumeFencedSubject(node string) string {
-	if node == "" {
-		return "ebs.fenced"
-	}
-	return "ebs." + node + ".fenced"
-}
-
-// VolumeFencedEvent announces that a node tore down a volume's export because
-// the lease moved. Winner is who holds it now, for the operator reading the
-// guest's stop reason.
-type VolumeFencedEvent struct {
-	Volume string `json:"volume"`
-	Node   string `json:"node"`
-	Winner string `json:"winner"`
-	Reason string `json:"reason"`
-}
-
-// VolumeFencedAck is the daemon's answer to a VolumeFencedEvent. Any answer
-// ends the sender's retries; Instance names the guest stopped, if one was, and
-// Error says why the daemon could not act on the event.
-type VolumeFencedAck struct {
-	Instance string `json:"instance,omitempty"`
-	Error    string `json:"error,omitempty"`
-}

contracts/viperblockd/legacy/v1/fence_test.go

@@ -1,54 +0,0 @@
-package viperblocklegacyv1_test
-
-import (
-	"encoding/json"
-	"testing"
-
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestVolumeFencedSubject_IsNodeAddressed pins the routing. A fenced guest is on
-// the node that lost the volume, so a subject without the node in it would reach
-// daemons with nothing to stop and, on a queue group, miss the one that has.
-func TestVolumeFencedSubject_IsNodeAddressed(t *testing.T) {
-	assert.Equal(t, "ebs.node-a.fenced", viperblocklegacyv1.VolumeFencedSubject("node-a"))
-	assert.Equal(t, "ebs.fenced", viperblocklegacyv1.VolumeFencedSubject(""),
-		"a single-node daemon has no node name, and still has to hear its own fences")
-}
-
-// TestVolumeFencedEvent_JSON pins the exact field spelling of the fenced
-// event, published by viperblockd and consumed by the daemon.
-func TestVolumeFencedEvent_JSON(t *testing.T) {
-	golden := `{"volume":"vol-1","node":"node-a","winner":"node-b","reason":"volume lease moved to node-b while this node had the volume open"}`
-	event := viperblocklegacyv1.VolumeFencedEvent{
-		Volume: "vol-1",
-		Node:   "node-a",
-		Winner: "node-b",
-		Reason: "volume lease moved to node-b while this node had the volume open",
-	}
-
-	marshaled, err := json.Marshal(event)
-	require.NoError(t, err)
-	assert.JSONEq(t, golden, string(marshaled))
-
-	var decoded viperblocklegacyv1.VolumeFencedEvent
-	require.NoError(t, json.Unmarshal([]byte(golden), &decoded))
-	assert.Equal(t, event, decoded)
-}
-
-// TestVolumeFencedAck_JSON pins the daemon's answer, which is what ends
-// viperblockd's retries of a fence notice.
-func TestVolumeFencedAck_JSON(t *testing.T) {
-	golden := `{"instance":"i-1","error":"bad event"}`
-	ack := viperblocklegacyv1.VolumeFencedAck{Instance: "i-1", Error: "bad event"}
-
-	marshaled, err := json.Marshal(ack)
-	require.NoError(t, err)
-	assert.JSONEq(t, golden, string(marshaled))
-
-	empty, err := json.Marshal(viperblocklegacyv1.VolumeFencedAck{})
-	require.NoError(t, err)
-	assert.JSONEq(t, `{}`, string(empty), "an ack for a volume with no local guest carries nothing")
-}

docs/DESIGN.md

@@ -112,14 +112,14 @@ When a daemon handles `RunInstances`:
 
 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
 2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
-3. **Volume Mount**: Sends a mount request to the colocated Viperblock worker on `ebs.{node}.mount`, receives NBD URI
+3. **Volume Mount**: Publishes each volume through the EBS provider contract (`PublishVolume` on `ebs.provider.v1.{node}.mount`), receives its export
 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management
 6. **Command subscription**: Subscribes to `ec2.cmd.{instanceID}` for subsequent commands on this VM
 
 ```go
-// Volume mounting via NATS (runtime/daemon/vm_adapters.go)
-reply, err := ebsRequestWithTrace(ctx, a.nc, instance.AccountID, viperblocklegacyv1.MountSubject(a.node), ebsMountRequest, 30*time.Second)
+// Volume mounting through the EBS provider contract (runtime/daemon/vm_adapters.go)
+published, err := a.provider.PublishVolume(ebsprovider.WithAccountID(ctx, accountID), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewLatestVersioned(), VolumeID: req.Name, NodeID: a.node})
 
 // QEMU launch with NBD-backed drives in args (runtime/compute/vm/vm.go)
 cmd := exec.Command(qemuArchitecture, args...) // qemu-system-x86_64 or qemu-system-aarch64

internal/testkit/drainsocket.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"sync/atomic"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -28,10 +29,11 @@ func SocketTempDir(t *testing.T) string {
 // StartDrainSocket stands in for the NBD plugin's drain socket, at the path the
 // plugin serves it on the node hosting volumeID ({dataDir}/viperblock/{volumeID}
 // /snapshot.sock). Every connection is answered with ack ("OK\n" for a drain
-// that reached S3, "ERR\n" for one that did not) and closed.
-func StartDrainSocket(t *testing.T, dataDir, volumeID, ack string) {
+// that reached S3, "ERR\n" for one that did not) and closed. connections
+// reports how many drains have been asked for so far.
+func StartDrainSocket(t *testing.T, dataDir, volumeID, ack string) (connections func() int) {
 	t.Helper()
-	serveDrainSocket(t, dataDir, volumeID, ack, nil, nil)
+	return serveDrainSocket(t, dataDir, volumeID, ack, nil, nil)
 }
 
 // StartBlockingDrainSocket stands in for a plugin whose flush is still running:
@@ -43,7 +45,7 @@ func StartBlockingDrainSocket(t *testing.T, dataDir, volumeID, ack string) (acce
 
 	acceptedCh := make(chan struct{})
 	releaseCh := make(chan struct{})
-	serveDrainSocket(t, dataDir, volumeID, ack, acceptedCh, releaseCh)
+	_ = serveDrainSocket(t, dataDir, volumeID, ack, acceptedCh, releaseCh)
 
 	var once sync.Once
 	t.Cleanup(func() { once.Do(func() { close(releaseCh) }) })
@@ -52,8 +54,9 @@ func StartBlockingDrainSocket(t *testing.T, dataDir, volumeID, ack string) (acce
 }
 
 // serveDrainSocket answers every connection with ack, optionally closing
-// accepted on the first one and waiting for release before it replies.
-func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan struct{}, release <-chan struct{}) {
+// accepted on the first one and waiting for release before it replies. The
+// returned func counts the connections accepted so far.
+func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan struct{}, release <-chan struct{}) func() int {
 	t.Helper()
 
 	dir := filepath.Join(dataDir, "viperblock", volumeID)
@@ -63,6 +66,7 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 	require.NoError(t, err)
 	t.Cleanup(func() { _ = ln.Close() })
 
+	var count atomic.Int64
 	go func() {
 		var once sync.Once
 		for {
@@ -70,6 +74,7 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 			if err != nil {
 				return // listener closed by the cleanup above
 			}
+			count.Add(1)
 			if accepted != nil {
 				once.Do(func() { close(accepted) })
 			}
@@ -80,4 +85,5 @@ func serveDrainSocket(t *testing.T, dataDir, volumeID, ack string, accepted chan
 			_ = conn.Close()
 		}
 	}()
+	return func() int { return int(count.Load()) }
 }

spinifex/domains/ec2/image/drain_test.go

@@ -1,140 +0,0 @@
-package image
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const drainImageInstanceID = "i-image-drain-host"
-
-// setupDrainImageService mirrors domains/ec2/snapshot's setupDrainService: a
-// reachable (but rejecting) Predastore host so a drained volume falls through
-// into the real snapshot path and fails fast, a data dir with no local drain
-// socket, and a NATS connection to route the drain to the hosting node.
-func setupDrainImageService(t *testing.T) (*ImageServiceImpl, *objectstore.MemoryObjectStore, *nats.Conn) {
-	t.Helper()
-
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	t.Cleanup(srv.Close)
-
-	_, nc := testutil.StartTestNATS(t)
-	store := objectstore.NewMemoryObjectStore()
-	cfg := &config.Config{
-		DataDir: testutil.SocketTempDir(t),
-		Predastore: config.PredastoreConfig{
-			Host:   srv.URL,
-			Bucket: testBucket,
-		},
-	}
-	svc := NewImageServiceImplWithConfig(cfg, store, nc)
-	return svc, store, nc
-}
-
-func seedImageVolumeAttachment(t *testing.T, store *objectstore.MemoryObjectStore, volumeID, state, instanceID string) {
-	t.Helper()
-	require.NoError(t, volumestate.Write(context.Background(), store, testBucket, volumeID, volumestate.Record{
-		State:            state,
-		AttachedInstance: instanceID,
-	}))
-}
-
-func drainImageResponder(t *testing.T, nc *nats.Conn, instanceID string, reply []byte) chan *nats.Msg {
-	t.Helper()
-
-	got := make(chan *nats.Msg, 4)
-	sub, err := nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
-		got <- msg
-		_ = msg.Respond(reply)
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-
-	return got
-}
-
-func drainedImageAck(t *testing.T, volumeID string) []byte {
-	t.Helper()
-	data, err := json.Marshal(ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusDrained})
-	require.NoError(t, err)
-	return data
-}
-
-func awaitImageDrainCommand(t *testing.T, got chan *nats.Msg) ec2v1.EC2InstanceCommand {
-	t.Helper()
-	select {
-	case msg := <-got:
-		var command ec2v1.EC2InstanceCommand
-		require.NoError(t, json.Unmarshal(msg.Data, &command))
-		return command
-	case <-time.After(2 * time.Second):
-		t.Fatal("the node hosting the volume never received a drain command")
-		return ec2v1.EC2InstanceCommand{}
-	}
-}
-
-// CreateImageFromInstance's IsRunning path must drain the volume's hosting node
-// before it reads the live checkpoint — the same guarantee ec2.CreateSnapshot
-// has. Without this, a checkpoint predating a guest-triggered GPT rewrite
-// (e.g. growpart on first boot) can be captured silently.
-func TestSnapshotRunningVolume_DrainsAttachedVolume(t *testing.T) {
-	svc, store, nc := setupDrainImageService(t)
-	createTestVolumeConfig(t, store, "vol-image-drain", 10)
-	seedImageVolumeAttachment(t, store, "vol-image-drain", "in-use", drainImageInstanceID)
-	got := drainImageResponder(t, nc, drainImageInstanceID, drainedImageAck(t, "vol-image-drain"))
-
-	err := svc.snapshotRunningVolume("vol-image-drain", "snap-image-drain", testAccountID)
-	// The rejecting Predastore stub fails the snapshot itself; what matters
-	// here is that the drain reached the hosting node first.
-	require.Error(t, err)
-
-	command := awaitImageDrainCommand(t, got)
-	assert.True(t, command.Attributes.DrainVolume)
-	assert.Equal(t, drainImageInstanceID, command.ID)
-	require.NotNil(t, command.DrainVolumeData)
-	assert.Equal(t, "vol-image-drain", command.DrainVolumeData.VolumeID)
-}
-
-// An attached volume whose host reports the drain failed must fail
-// CreateImage rather than register an AMI built from a stale checkpoint.
-func TestSnapshotRunningVolume_UndrainableAttachedVolumeFails(t *testing.T) {
-	svc, store, nc := setupDrainImageService(t)
-	createTestVolumeConfig(t, store, "vol-image-nodrain", 10)
-	seedImageVolumeAttachment(t, store, "vol-image-nodrain", "in-use", drainImageInstanceID)
-	drainImageResponder(t, nc, drainImageInstanceID,
-		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
-
-	err := svc.snapshotRunningVolume("vol-image-nodrain", "snap-image-nodrain", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// An unattached (stopped-instance) volume takes the no-drain path: the
-// checkpoint Close() left behind is already current.
-func TestSnapshotRunningVolume_AvailableSkipsDrain(t *testing.T) {
-	svc, store, _ := setupDrainImageService(t)
-	createTestVolumeConfig(t, store, "vol-image-available", 10)
-	seedImageVolumeAttachment(t, store, "vol-image-available", "available", "")
-	svc.natsConn = nil
-
-	// No responder exists and the socket is never dialled: proceeding straight
-	// to the (rejecting) snapshot path proves the drain was skipped, not stuck.
-	err := svc.snapshotRunningVolume("vol-image-available", "snap-image-available", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}

spinifex/domains/ec2/image/provider_snapshot_test.go

@@ -2,16 +2,20 @@ package image
 
 import (
 	"context"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -96,15 +100,15 @@ func TestGetVolumeMetadata_Provider_UnknownVolumeIsNotFound(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error())
 }
 
-// TestSnapshotStoppedVolume_Provider_CreatesThroughProvider locks that a
-// stopped-volume snapshot under a provider is created via provider.CreateSnapshot
+// TestSnapshotVolume_Provider_CreatesThroughProvider locks that a
+// root-volume snapshot under a provider is created via provider.CreateSnapshot
 // and never falls through to building an embedded engine — the Predastore host
 // is a closed port, so any fallback would fail with connection refused.
-func TestSnapshotStoppedVolume_Provider_CreatesThroughProvider(t *testing.T) {
+func TestSnapshotVolume_Provider_CreatesThroughProvider(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-stopped001", 8)
 
-	err := svc.snapshotStoppedVolume(doc.VolumeID, "snap-stopped001", testAccountID)
+	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped001", testAccountID)
 	require.NoError(t, err, "the provider path must not fall through to the embedded engine against the closed Predastore port")
 
 	// CreateSnapshot is idempotent on (SnapshotID, VolumeID): re-issuing it
@@ -118,22 +122,22 @@ func TestSnapshotStoppedVolume_Provider_CreatesThroughProvider(t *testing.T) {
 	assert.Equal(t, doc.VolumeID, snap.SourceVolumeID)
 }
 
-// TestSnapshotRunningVolume_Provider_PassesProviderHandleAsVolumeHandle locks
+// TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle locks
 // that the ebsmetadata document's ProviderHandle is threaded through as
 // CreateSnapshotRequest.VolumeHandle. A wrong handle must be rejected by the
 // provider (MemoryProvider treats an empty VolumeHandle as "skip the check",
 // so only a mismatched non-empty value proves the real value is forwarded).
-func TestSnapshotRunningVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
+func TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-running001", 8)
 
-	err := svc.snapshotRunningVolume(doc.VolumeID, "snap-running001", testAccountID)
+	err := svc.snapshotVolume(doc.VolumeID, "snap-running001", testAccountID)
 	require.NoError(t, err, "the correct provider handle must let CreateSnapshot succeed")
 
 	doc.ProviderHandle = "wrong-handle"
 	require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc))
 
-	err = svc.snapshotRunningVolume(doc.VolumeID, "snap-running002", testAccountID)
+	err = svc.snapshotVolume(doc.VolumeID, "snap-running002", testAccountID)
 	require.Error(t, err, "a mismatched provider handle must be rejected, proving the value came from the document")
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
@@ -222,3 +226,98 @@ func TestCreateImageFromInstance_SetsEnaSupportFromInstance(t *testing.T) {
 	require.NotNil(t, ami.EnaSupport)
 	assert.True(t, *ami.EnaSupport)
 }
+
+// createUnavailableProvider refuses every CreateSnapshot as retry-later.
+type createUnavailableProvider struct {
+	ebsprovider.EBSProvider
+}
+
+func (p *createUnavailableProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}
+}
+
+// TestSnapshotVolume_Provider_UnavailableIsRetryLater locks that the
+// CreateImage snapshot surfaces a provider's retry-later as ServiceUnavailable.
+func TestSnapshotVolume_Provider_UnavailableIsRetryLater(t *testing.T) {
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	doc := seedProviderVolume(t, svc, provider, "vol-stopped-busy", 8)
+	svc.SetEBSProvider(&createUnavailableProvider{EBSProvider: provider})
+
+	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped-busy", testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServiceUnavailable, err.Error())
+}
+
+// createInUseProvider refuses every CreateSnapshot as volume in use.
+type createInUseProvider struct {
+	ebsprovider.EBSProvider
+}
+
+func (p *createInUseProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "published"}
+}
+
+// TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState locks that the
+// CreateImage snapshot tells the caller to detach a volume the provider refuses as in use.
+func TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState(t *testing.T) {
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	doc := seedProviderVolume(t, svc, provider, "vol-attached-inuse", 8)
+	svc.SetEBSProvider(&createInUseProvider{EBSProvider: provider})
+
+	err := svc.snapshotVolume(doc.VolumeID, "snap-attached-inuse", testAccountID)
+
+	code, message, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "unregistered error: %v", err)
+	assert.Equal(t, awserrors.ErrorIncorrectState, code)
+	assert.Contains(t, message, doc.VolumeID)
+	assert.Contains(t, message, "Detach")
+}
+
+// countingSnapshotProvider counts the CreateSnapshot calls it forwards.
+type countingSnapshotProvider struct {
+	ebsprovider.EBSProvider
+
+	creates atomic.Int32
+}
+
+func (p *countingSnapshotProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	p.creates.Add(1)
+	return p.EBSProvider.CreateSnapshot(ctx, req)
+}
+
+// An image of a running instance is snapshotted by the provider, which drains
+// a buffered volume itself, so CreateImage neither dials the drain socket nor
+// asks the instance's host to drain.
+func TestCreateImageFromInstance_RunningInstanceLeavesTheDrainToTheProvider(t *testing.T) {
+	const instanceID = "i-running-drain"
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	_, nc := testutil.StartTestNATS(t)
+	svc.natsConn = nc
+	svc.config.DataDir = testutil.SocketTempDir(t)
+	counting := &countingSnapshotProvider{EBSProvider: provider}
+	svc.SetEBSProvider(counting)
+
+	doc := seedProviderVolume(t, svc, provider, "vol-running-drain", 8)
+	doc.State, doc.AttachedInstance = "in-use", instanceID
+	require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc))
+	drains := testutil.StartDrainSocket(t, svc.config.DataDir, doc.VolumeID, "OK\n")
+	var commands atomic.Int32
+	sub, err := nc.Subscribe(ec2v1.InstanceCommandSubject(instanceID), func(msg *nats.Msg) {
+		commands.Add(1)
+		_ = msg.Respond([]byte(`{"Code":"InternalError","Message":"no drain socket"}`))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	out, err := svc.CreateImageFromInstance(CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String(instanceID), Name: aws.String("running-ami")},
+		RootVolumeID: doc.VolumeID,
+		IsRunning:    true,
+	}, testAccountID)
+	require.NoError(t, err)
+	require.NotNil(t, out.ImageId)
+
+	assert.Equal(t, int32(1), counting.creates.Load(), "the provider must take the snapshot")
+	assert.Zero(t, drains(), "CreateImage must not dial the drain socket")
+	assert.Zero(t, commands.Load(), "CreateImage must not ask the instance's host to drain")
+}

spinifex/domains/ec2/image/service_impl.go

@@ -38,7 +38,7 @@ type CreateImageParams struct {
 	Input         *ec2.CreateImageInput
 	RootVolumeID  string
 	SourceImageID string
-	IsRunning     bool // true = use live checkpoint (instance still running), false = use numbered checkpoint from Close
+	IsRunning     bool // logged only: the provider's CreateSnapshot drains a live volume itself
 }
 
 // ImageServiceImpl handles AMI image operations with S3 storage.
@@ -69,9 +69,7 @@ func (s *ImageServiceImpl) MetadataStore() *ebsmetadata.Store {
 	return s.metadata
 }
 
-// NewImageServiceImpl creates a new daemon-side image service. natsConn is used
-// to drain a running instance's volume (routed to the node hosting it) before
-// CreateImageFromInstance reads its live checkpoint.
+// NewImageServiceImpl creates a new daemon-side image service.
 func NewImageServiceImpl(cfg *config.Config, natsConn *nats.Conn) *ImageServiceImpl {
 	store := objectstore.NewS3ObjectStoreFromConfig(
 		cfg.Predastore.Host,
@@ -99,8 +97,7 @@ func NewImageServiceImplWithStore(store objectstore.ObjectStore, bucketName stri
 }
 
 // NewImageServiceImplWithConfig creates an image service with an explicit
-// config and NATS connection (for testing the running-volume drain path,
-// which needs config.DataDir/Predastore and a NATS connection to route to).
+// config, object store and NATS connection (for testing).
 func NewImageServiceImplWithConfig(cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn) *ImageServiceImpl {
 	return &ImageServiceImpl{
 		config:     cfg,
@@ -447,15 +444,9 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		"rootVolumeId", params.RootVolumeID, "amiId", amiID, "snapshotId", snapshotID,
 		"isRunning", params.IsRunning)
 
-	// Step 1: Snapshot root volume (live via NATS or offline from S3)
-	var snapshotErr error
-	if params.IsRunning {
-		snapshotErr = s.snapshotRunningVolume(params.RootVolumeID, snapshotID, accountID)
-	} else {
-		snapshotErr = s.snapshotStoppedVolume(params.RootVolumeID, snapshotID, accountID)
-	}
-	if snapshotErr != nil {
-		return nil, snapshotErr
+	// Step 1: Snapshot root volume
+	if err := s.snapshotVolume(params.RootVolumeID, snapshotID, accountID); err != nil {
+		return nil, err
 	}
 
 	// Step 2: Read source volume config for size
@@ -521,51 +512,21 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 	}, nil
 }
 
-// snapshotRunningVolume creates a snapshot of a running instance's volume by
-// draining it to the node hosting it (so any writes buffered there reach S3),
-// then reading the live checkpoint written by nbdkit on every NBD Flush. This
-// shares domains/ec2/snapshot's DrainVolume with ec2.CreateSnapshot so the two
-// live-snapshot paths cannot diverge on when a drain is required.
-func (s *ImageServiceImpl) snapshotRunningVolume(volumeID, snapshotID, accountID string) error {
+// snapshotVolume snapshots the root volume through the EBS provider, whether
+// or not the instance is running. A provider that buffers writes drains the
+// volume inside its own CreateSnapshot, so no drain is issued here.
+func (s *ImageServiceImpl) snapshotVolume(volumeID, snapshotID, accountID string) error {
 	if s.config == nil {
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
 	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, volumeID)
 	if err != nil {
-		slog.Error("snapshotRunningVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
+		slog.Error("snapshotVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// Flush the writes still buffered by whichever node serves the volume, so
-	// the snapshot below is current rather than an arbitrarily stale one
-	// predating e.g. a guest-triggered GPT rewrite (growpart on first boot). An
-	// attached volume that cannot be drained fails here rather than silently
-	// producing an image from a stale checkpoint.
-	if err := ec2snapshot.DrainVolume(context.Background(), s.config, s.store, s.natsConn, volumeID, volMeta.State, volMeta.AttachedInstance, accountID); err != nil {
-		slog.Error("snapshotRunningVolume: drain failed", "volumeId", volumeID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotRunningVolume")
-}
-
-// snapshotStoppedVolume creates a snapshot of a detached volume. No drain: the
-// volume is stopped/detached, so nothing is buffered on a serving node.
-// viperblockd's handleCreateSnapshot prefers a live mounted engine over opening
-// a second one, so it handles the mounted case itself.
-func (s *ImageServiceImpl) snapshotStoppedVolume(volumeID, snapshotID, accountID string) error {
-	if s.config == nil {
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, volumeID)
-	if err != nil {
-		slog.Error("snapshotStoppedVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotStoppedVolume")
+	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotVolume")
 }
 
 // createSnapshot delegates snapshot creation for volMeta to the EBS provider.
@@ -580,7 +541,7 @@ func (s *ImageServiceImpl) createSnapshot(ctx context.Context, volMeta ebsmetada
 	})
 	if err != nil || created == nil {
 		slog.ErrorContext(ctx, "provider CreateSnapshot failed", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
+		return ec2snapshot.ProviderCreateError(volMeta.VolumeID, err)
 	}
 	slog.InfoContext(ctx, "snapshot created", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID)
 	return nil

spinifex/domains/ec2/image/service_impl_test.go

@@ -2551,56 +2551,19 @@ func TestResetImageAttribute_MissingParameters(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorMissingParameter, err.Error())
 }
 
-func TestSnapshotRunningVolume_VolumeConfigMissing(t *testing.T) {
+func TestSnapshotVolume_VolumeConfigMissing(t *testing.T) {
 	store := objectstore.NewMemoryObjectStore()
 	svc := &ImageServiceImpl{
 		config:     &config.Config{},
 		store:      store,
 		bucketName: testBucket,
 	}
-	err := svc.snapshotRunningVolume("vol-missing", "snap-r1", testAccountID)
+	err := svc.snapshotVolume("vol-missing", "snap-s1", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
-func TestSnapshotRunningVolume_BackendInitFails(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:   srv.URL,
-				Region: "us-east-1",
-				Bucket: testBucket,
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-running1", 10)
-
-	err := svc.snapshotRunningVolume("vol-running1", "snap-r2", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotStoppedVolume_VolumeConfigMissing(t *testing.T) {
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config:     &config.Config{},
-		store:      store,
-		bucketName: testBucket,
-	}
-	err := svc.snapshotStoppedVolume("vol-missing", "snap-s1", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotStoppedVolume_BackendInitFails(t *testing.T) {
+func TestSnapshotVolume_BackendInitFails(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusForbidden)
 	}))
@@ -2620,7 +2583,7 @@ func TestSnapshotStoppedVolume_BackendInitFails(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped1", 10)
 
-	err := svc.snapshotStoppedVolume("vol-stopped1", "snap-s2", testAccountID)
+	err := svc.snapshotVolume("vol-stopped1", "snap-s2", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
@@ -2646,60 +2609,7 @@ func imageS3MockServer(t *testing.T) *httptest.Server {
 	}))
 }
 
-func TestSnapshotRunningVolume_EncryptionKeyLoadError(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-	}))
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:   srv.URL,
-				Region: "us-east-1",
-				Bucket: testBucket,
-			},
-			Viperblock: config.ViperblockConfig{
-				EncryptionKeyFile: "/nonexistent-image-run-key.bin",
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-running-enckey", 10)
-
-	err := svc.snapshotRunningVolume("vol-running-enckey", "snap-run-enckey", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotRunningVolume_LoadStateFails(t *testing.T) {
-	srv := imageS3MockServer(t)
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:      srv.URL,
-				Region:    "us-east-1",
-				Bucket:    testBucket,
-				AccessKey: "test-key",
-				SecretKey: "test-secret",
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-running-lsf", 10)
-
-	err := svc.snapshotRunningVolume("vol-running-lsf", "snap-run-lsf", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotStoppedVolume_EncryptionKeyLoadError(t *testing.T) {
+func TestSnapshotVolume_EncryptionKeyLoadError(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 	}))
@@ -2722,12 +2632,12 @@ func TestSnapshotStoppedVolume_EncryptionKeyLoadError(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped-enckey", 10)
 
-	err := svc.snapshotStoppedVolume("vol-stopped-enckey", "snap-stop-enckey", testAccountID)
+	err := svc.snapshotVolume("vol-stopped-enckey", "snap-stop-enckey", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
-func TestSnapshotStoppedVolume_LoadStateFails(t *testing.T) {
+func TestSnapshotVolume_LoadStateFails(t *testing.T) {
 	srv := imageS3MockServer(t)
 	defer srv.Close()
 
@@ -2747,7 +2657,7 @@ func TestSnapshotStoppedVolume_LoadStateFails(t *testing.T) {
 	}
 	createTestVolumeConfig(t, store, "vol-stopped-lsf", 10)
 
-	err := svc.snapshotStoppedVolume("vol-stopped-lsf", "snap-stop-lsf", testAccountID)
+	err := svc.snapshotVolume("vol-stopped-lsf", "snap-stop-lsf", testAccountID)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }

spinifex/domains/ec2/instance/service_impl.go

@@ -2689,15 +2689,6 @@ func (s *InstanceServiceImpl) capacityRefusal(ctx context.Context, instance *vm.
 // start returns it to the stopped store, a recovery returns it to the node it
 // was taken from.
 func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instance *vm.VM, undo func()) error {
-	// Before anything is stamped or claimed, so the record handed back is the
-	// one that was claimed.
-	if err := vm.CheckUEFIVarStore(instance); err != nil {
-		slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without exactly one variable store",
-			"instanceId", instance.ID, "err", err)
-		undo()
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
 	instance.ResetNodeLocalState()
 
 	// AWS restamps LaunchTime on every start. The pending watchdog times a

spinifex/domains/ec2/instance/service_impl_test.go

@@ -2605,34 +2605,6 @@ func TestStartStoppedInstance_InstanceTypeUnknown(t *testing.T) {
 	assert.Equal(t, vm.StateStopped, store.WroteStopped[id].Status)
 }
 
-// A UEFI instance with no variable store pointer is refused before anything is
-// stamped or claimed, so the record handed back is the one that was claimed.
-func TestStartStoppedInstance_UEFIInstanceWithoutAPointerIsRefused(t *testing.T) {
-	id := "i-uefi-nopointer"
-	itype := "t3.micro"
-	launched := time.Unix(1700000000, 0).UTC()
-	stopped := &vm.VM{
-		ID: id, Status: vm.StateStopped, AccountID: "acc", InstanceType: itype, BootMode: "uefi",
-		Instance: &ec2.Instance{LaunchTime: aws.Time(launched)},
-	}
-	before := stopped.Record()
-	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stopped}}
-	prov := &fakeResourceCapacityProvider{instanceTypes: map[string]*ec2.InstanceTypeInfo{
-		itype: {InstanceType: aws.String(itype)},
-	}}
-	svc := &InstanceServiceImpl{stoppedStore: store, resourceMgr: prov, vmMgr: vm.NewManager()}
-
-	_, err := svc.StartStoppedInstance(context.Background(), &StartStoppedInstanceInput{InstanceID: id}, "acc")
-
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-	assert.Empty(t, prov.allocated, "nothing is allocated for a start that cannot boot")
-	require.NotNil(t, store.WroteStopped[id], "the claimed instance is handed back")
-	assert.Equal(t, before, store.WroteStopped[id].Record(), "the record is unchanged")
-	_, inManager := svc.vmMgr.Get(id)
-	assert.False(t, inManager)
-}
-
 func TestStartStoppedInstance_AllocateFails(t *testing.T) {
 	id := "i-alloc-fail"
 	itype := "t3.micro"

spinifex/domains/ec2/snapshot/drain_test.go

@@ -2,12 +2,8 @@ package snapshot
 
 import (
 	"context"
-	"encoding/json"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"net/http"
-	"net/http/httptest"
+	"sync/atomic"
 	"testing"
-	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -15,8 +11,6 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go"
@@ -24,321 +18,78 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const drainInstanceID = "i-drain-host"
+const attachedInstanceID = "i-attached-host"
 
-// setupDrainService returns a snapshot service configured like a daemon that
-// answers ec2.CreateSnapshot: a reachable Predastore host (so drains are not
-// skipped as metadata-only), a data dir with no drain socket in it, and a NATS
-// connection. The Predastore stub rejects everything, so a snapshot that gets
-// past the drain fails fast rather than hanging on a dead address.
-func setupDrainService(t *testing.T) (*SnapshotServiceImpl, *objectstore.MemoryObjectStore, *nats.Conn) {
+// countingSnapshotProvider counts the CreateSnapshot calls it forwards.
+type countingSnapshotProvider struct {
+	ebsprovider.EBSProvider
+
+	creates atomic.Int32
+}
+
+func (p *countingSnapshotProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	p.creates.Add(1)
+	return p.EBSProvider.CreateSnapshot(ctx, req)
+}
+
+// setupAttachedSnapshot is a viperblockd node answering CreateSnapshot for a
+// volume attached to a running instance, with a memory provider behind it.
+func setupAttachedSnapshot(t *testing.T, volumeID string) (*SnapshotServiceImpl, *countingSnapshotProvider, *nats.Conn) {
 	t.Helper()
 
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	t.Cleanup(srv.Close)
-
 	_, nc := testutil.StartTestNATS(t)
-	store := objectstore.NewMemoryObjectStore()
 	cfg := &config.Config{
-		DataDir: testutil.SocketTempDir(t),
-		Predastore: config.PredastoreConfig{
-			Host:   srv.URL,
-			Bucket: "test-bucket",
-		},
+		DataDir:    testutil.SocketTempDir(t),
+		EBS:        config.EBSConfig{Provider: config.EBSProviderViperblockd},
+		Predastore: config.PredastoreConfig{Host: "127.0.0.1:1", Bucket: "test-bucket"},
 	}
-	return NewSnapshotServiceImplWithStore(cfg, store, nc), store, nc
-}
+	svc := NewSnapshotServiceImplWithStore(cfg, objectstore.NewMemoryObjectStore(), nc)
+	provider := &countingSnapshotProvider{EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})}
+	svc.SetEBSProvider(provider)
 
-// seedVolumeAttachment writes the control-plane state.json that decides whether
-// a snapshot must drain.
-func seedVolumeAttachment(t *testing.T, store *objectstore.MemoryObjectStore, volumeID, state, instanceID string) {
-	t.Helper()
-	require.NoError(t, volumestate.Write(context.Background(), store, "test-bucket", volumeID, volumestate.Record{
-		State:            state,
-		AttachedInstance: instanceID,
-	}))
-}
-
-// drainResponder answers the per-instance command subject the way the node
-// hosting the instance does, and hands each request it received to the test.
-func drainResponder(t *testing.T, nc *nats.Conn, instanceID string, reply []byte) chan *nats.Msg {
-	t.Helper()
-
-	got := make(chan *nats.Msg, 4)
-	sub, err := nc.Subscribe("ec2.cmd."+instanceID, func(msg *nats.Msg) {
-		got <- msg
-		_ = msg.Respond(reply)
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-
-	return got
-}
-
-// drainedAck is the reply of a node whose drain reached S3.
-func drainedAck(t *testing.T, volumeID string) []byte {
-	t.Helper()
-	data, err := json.Marshal(ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusDrained})
-	require.NoError(t, err)
-	return data
-}
-
-// requireReturnsWithin fails the test if fn has not returned successfully by
-// limit, so a drain that blocks on a socket it should not have dialed reports
-// as a failure rather than as a slow test.
-func requireReturnsWithin(t *testing.T, limit time.Duration, fn func() error) {
-	t.Helper()
-
-	done := make(chan error, 1)
-	go func() { done <- fn() }()
-
-	select {
-	case err := <-done:
-		require.NoError(t, err)
-	case <-time.After(limit):
-		t.Fatalf("drain did not return within %s", limit)
-	}
-}
-
-// awaitDrainCommand returns the command the hosting node received, failing the
-// test if none arrives.
-func awaitDrainCommand(t *testing.T, got chan *nats.Msg) ec2v1.EC2InstanceCommand {
-	t.Helper()
-	select {
-	case msg := <-got:
-		var command ec2v1.EC2InstanceCommand
-		require.NoError(t, json.Unmarshal(msg.Data, &command))
-		assert.Equal(t, testAccountID, msg.Header.Get(natsmsg.AccountIDHeader))
-		return command
-	case <-time.After(2 * time.Second):
-		t.Fatal("the node hosting the volume never received a drain command")
-		return ec2v1.EC2InstanceCommand{}
-	}
-}
-
-// An attached volume is drained on the node hosting its instance, which is the
-// only node with the socket — the node answering CreateSnapshot usually is not.
-func TestDrainVolume_AttachedRoutesToHostingNode(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-attached", "in-use", drainInstanceID)
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-attached"))
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-attached", "", "", testAccountID))
-
-	command := awaitDrainCommand(t, got)
-	assert.True(t, command.Attributes.DrainVolume)
-	assert.Equal(t, drainInstanceID, command.ID)
-	require.NotNil(t, command.DrainVolumeData)
-	assert.Equal(t, "vol-attached", command.DrainVolumeData.VolumeID)
-}
-
-// No responder on the command subject means no node is running the instance,
-// which is what a stopped instance looks like: stop tears down both the plugin
-// and the subscription but deliberately leaves a boot volume attached. Failing
-// here would make every stopped instance's root volume unsnapshottable.
-func TestDrainVolume_AttachedToStoppedInstanceTakesStoppedPath(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-no-host", "in-use", drainInstanceID)
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-no-host", "", "", testAccountID))
-}
-
-// A host that still holds the instance but reports it not running has nothing
-// to flush, so the sealed checkpoint stands. This is the host-drain stop, which
-// unmounts the volumes but keeps the subscription.
-func TestDrainVolume_NotRunningAckTakesStoppedPath(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-not-running", "in-use", drainInstanceID)
-	ack, err := json.Marshal(ec2v1.DrainVolumeResponse{
-		VolumeID: "vol-not-running", Status: ec2v1.DrainVolumeStatusNotRunning,
-	})
-	require.NoError(t, err)
-	got := drainResponder(t, nc, drainInstanceID, ack)
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-not-running", "", "", testAccountID))
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-}
-
-// A hosting node that reports the drain failed (wedged plugin, socket gone
-// mid-restart) fails the snapshot rather than falling through.
-func TestDrainVolume_AttachedWithFailedAckFails(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-drain-err", "in-use", drainInstanceID)
-	drainResponder(t, nc, drainInstanceID,
-		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
-
-	err := svc.drainVolume(context.Background(), "vol-drain-err", "", "", testAccountID)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "drain failed")
-}
-
-// An ack that is not the drained ack is not evidence the writes reached S3.
-func TestDrainVolume_AttachedWithUnexpectedAckFails(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-odd-ack", "in-use", drainInstanceID)
-	drainResponder(t, nc, drainInstanceID, []byte(`{}`))
-
-	err := svc.drainVolume(context.Background(), "vol-odd-ack", "", "", testAccountID)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "unexpected ack")
-}
-
-// Nothing writes to an unattached volume, so the checkpoint its Close() left
-// behind is current and no drain is attempted — not even locally. Dialing the
-// socket is not a probe: it makes the plugin run a full flush, so an available
-// volume must be decided from the record without touching it.
-func TestDrainVolume_AvailableDoesNotDialLocalSocket(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-available", "available", "")
-
-	// A socket that never answers: a dial would park here until the read
-	// deadline, so returning promptly is what proves none was made.
-	testutil.StartBlockingDrainSocket(t, svc.config.DataDir, "vol-available", "OK\n")
-
-	// A nil connection proves no command was issued: routing one would error.
-	svc.natsConn = nil
-
-	requireReturnsWithin(t, 2*time.Second, func() error {
-		return svc.drainVolume(context.Background(), "vol-available", "", "", testAccountID)
-	})
-}
-
-// A volume recorded as in-use but with no instance (drifted state) has no node
-// to address, so it takes the stopped path rather than failing every snapshot.
-func TestDrainVolume_InUseWithoutInstanceTakesStoppedPath(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-no-instance", "in-use", "")
-	svc.natsConn = nil
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-no-instance", "", "", testAccountID))
-}
-
-// When the volume is served by this node the socket is local, so the drain
-// short-circuits without a hop. This is every single-node deployment.
-func TestDrainVolume_LocalSocketShortCircuits(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-local", "in-use", drainInstanceID)
-	testutil.StartDrainSocket(t, svc.config.DataDir, "vol-local", "OK\n")
-
-	// A nil connection proves the local ack was enough: no responder exists for
-	// drainInstanceID, so a routed command would fail.
-	svc.natsConn = nil
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-local", "", "", testAccountID))
-}
-
-// A socket that answers without acking means this node does serve the volume
-// and the flush failed. Routing that onward would come back to this same node
-// and re-run the drain that has already failed, so it fails here instead.
-func TestDrainVolume_LocalSocketErrorAckFailsWithoutRouting(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-local-err", "in-use", drainInstanceID)
-	testutil.StartDrainSocket(t, svc.config.DataDir, "vol-local-err", "ERR\n")
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-local-err"))
-
-	err := svc.drainVolume(context.Background(), "vol-local-err", "", "", testAccountID)
-	require.Error(t, err)
-
-	assert.Empty(t, got, "a local drain that failed must not be re-issued to the host node")
-}
-
-// state.json is the control-plane-owned attachment record; the copy inside
-// config.json is rewritten by the live NBD plugin from its stale in-memory
-// state, so a volume reading "available" there can still be attached.
-func TestDrainVolume_StateJSONOverridesConfigJSON(t *testing.T) {
-	svc, store, nc := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-stale-config", "in-use", drainInstanceID)
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-stale-config"))
-
-	// "available" mirrors the stale State a live NBD plugin leaves in
-	// config.json; drainVolume must not trust it (see the test's own doc above).
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-stale-config", "available", "", testAccountID))
-
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-}
-
-// Volumes predating the state.json split have no state object, so the state
-// embedded in config.json is all there is to decide from.
-func TestDrainVolume_FallsBackToConfigJSONWhenNoStateObject(t *testing.T) {
-	svc, _, nc := setupDrainService(t)
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-legacy"))
-
-	// The state/instance embedded in a legacy config.json, with no state.json to overlay it.
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-legacy", "in-use", drainInstanceID, testAccountID))
-
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-}
-
-// Without Predastore the snapshot is metadata-only and never reads a live
-// checkpoint, so there is nothing for a drain to make current.
-func TestDrainVolume_MetadataOnlySkipsDrain(t *testing.T) {
-	svc, store, _ := setupDrainService(t)
-	seedVolumeAttachment(t, store, "vol-meta-only", "in-use", drainInstanceID)
-	svc.config.Predastore.Host = ""
-	svc.natsConn = nil
-
-	require.NoError(t, svc.drainVolume(context.Background(), "vol-meta-only", "", "", testAccountID))
-}
-
-// putProviderVolume seeds both the control-plane document CreateSnapshot's
-// provider branch reads for attachment state, and the provider's own record,
-// mirroring what CreateVolume's provider branch leaves behind.
-func putProviderVolume(t *testing.T, svc *SnapshotServiceImpl, provider ebsprovider.EBSProvider, volumeID, state, instanceID string) {
-	t.Helper()
-	require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{
-		VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4,
-		State: state, AttachedInstance: instanceID, AvailabilityZone: "us-east-1a",
-		ProviderHandle: "memory://volume/" + volumeID,
-	}))
 	_, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID,
 		CapacityRange:    ebsprovider.CapacityRange{RequiredBytes: 4 * 1024 * 1024 * 1024},
 		AvailabilityZone: "us-east-1a",
 	})
 	require.NoError(t, err)
+	require.NoError(t, svc.metadata.PutVolume(context.Background(), ebsmetadata.Volume{
+		VolumeID: volumeID, TenantID: testAccountID, CapacityGiB: 4,
+		State: "in-use", AttachedInstance: attachedInstanceID, AvailabilityZone: "us-east-1a",
+	}))
+	return svc, provider, nc
 }
 
-// The provider branch used to skip the drain entirely, silently snapshotting a
-// stale checkpoint of an attached, actively-written volume. It must drain the
-// same way the legacy branch does.
-func TestCreateSnapshot_Provider_DrainsAttachedVolume(t *testing.T) {
-	svc, _, nc := setupDrainService(t)
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
-	svc.SetEBSProvider(provider)
-	putProviderVolume(t, svc, provider, "vol-provider-drain", "in-use", drainInstanceID)
-
-	got := drainResponder(t, nc, drainInstanceID, drainedAck(t, "vol-provider-drain"))
+// The provider drains a buffered volume inside its own CreateSnapshot, so the
+// domain must not dial the drain socket itself: that would drain twice.
+func TestCreateSnapshot_AttachedVolumeOpensNoDrainSocket(t *testing.T) {
+	svc, provider, _ := setupAttachedSnapshot(t, "vol-attached-local")
+	drains := testutil.StartDrainSocket(t, svc.config.DataDir, "vol-attached-local", "OK\n")
 
 	_, err := svc.CreateSnapshot(context.Background(),
-		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider-drain")}, testAccountID)
+		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-attached-local")}, testAccountID)
 	require.NoError(t, err)
 
-	command := awaitDrainCommand(t, got)
-	assert.True(t, command.Attributes.DrainVolume)
-	assert.Equal(t, "vol-provider-drain", command.DrainVolumeData.VolumeID)
+	assert.Zero(t, drains(), "the domain must leave the drain to the provider")
+	assert.Equal(t, int32(1), provider.creates.Load())
 }
 
-// A provider-branch snapshot whose drain fails must fail closed rather than
-// return a snapshot of stale data: the provider must never be asked to
-// snapshot, and no snapshot metadata may be left behind.
-func TestCreateSnapshot_Provider_UndrainableAttachedVolumeFails(t *testing.T) {
-	svc, _, nc := setupDrainService(t)
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
-	svc.SetEBSProvider(provider)
-	putProviderVolume(t, svc, provider, "vol-provider-nodrain", "in-use", drainInstanceID)
-
-	got := drainResponder(t, nc, drainInstanceID,
-		[]byte(`{"Code":"`+awserrors.ErrorServerInternal+`","Message":"drain failed"}`))
-
-	_, err := svc.CreateSnapshot(context.Background(),
-		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-provider-nodrain")}, testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-	assert.True(t, awaitDrainCommand(t, got).Attributes.DrainVolume)
-
-	out, err := svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{}, testAccountID)
+// With no drain socket on this node, an attached volume is snapshotted by the
+// provider without asking the instance's host for a drain.
+func TestCreateSnapshot_AttachedVolumeSendsItsHostNoDrain(t *testing.T) {
+	svc, provider, nc := setupAttachedSnapshot(t, "vol-attached-remote")
+	var commands atomic.Int32
+	sub, err := nc.Subscribe(ec2v1.InstanceCommandSubject(attachedInstanceID), func(msg *nats.Msg) {
+		commands.Add(1)
+		_ = msg.Respond([]byte(`{"Code":"InternalError","Message":"no drain socket"}`))
+	})
 	require.NoError(t, err)
-	assert.Empty(t, out.Snapshots, "a failed drain must not leave a snapshot behind")
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	snap, err := svc.CreateSnapshot(context.Background(),
+		&ec2.CreateSnapshotInput{VolumeId: aws.String("vol-attached-remote")}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, "vol-attached-remote", aws.StringValue(snap.VolumeId))
+	assert.Equal(t, int32(1), provider.creates.Load())
+	assert.Zero(t, commands.Load(), "the instance's host must not be asked to drain")
 }

spinifex/domains/ec2/snapshot/service_impl.go

@@ -5,10 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"log/slog"
-	"net"
-	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -18,10 +15,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/volumestate"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
@@ -209,19 +204,12 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
 		return nil, errors.New(awserrors.ErrorIncorrectState)
 	}
 
-	// Flush the writes buffered by whichever node serves the volume, so the
-	// provider reads a current checkpoint. An attached volume that cannot be
-	// drained fails here rather than silently snapshotting stale data.
-	if err := s.drainVolume(ctx, volumeID, volume.State, volume.AttachedInstance, accountID); err != nil {
-		slog.ErrorContext(ctx, "CreateSnapshot: drain failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-
 	created, err := s.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
 		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID, VolumeHandle: volume.ProviderHandle,
 	})
 	if err != nil || created == nil {
-		return nil, errors.New(awserrors.ErrorServerInternal)
+		slog.ErrorContext(ctx, "CreateSnapshot: provider CreateSnapshot failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+		return nil, ProviderCreateError(volumeID, err)
 	}
 	now := time.Now()
 	snapshotCfg := ebsmetadata.Snapshot{
@@ -259,177 +247,6 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
 	return snapshotConfigToEC2(snapshotCfg), nil
 }
 
-const (
-	// drainDialTimeout bounds the connect to the local drain socket. The socket
-	// is either being served on this node or it is not, so a slow connect means
-	// a wedged plugin rather than a busy one.
-	drainDialTimeout = time.Second
-
-	// drainAckTimeout bounds DrainToBackend on the hosting node: it flushes the
-	// WAL and the live checkpoint to S3, so it scales with the dirty set rather
-	// than with a fixed unit of work.
-	drainAckTimeout = 30 * time.Second
-
-	// drainRequestTimeout bounds the ec2.cmd round-trip. It exceeds
-	// drainAckTimeout so a slow drain surfaces as the hosting node's error
-	// instead of an opaque NATS timeout here.
-	drainRequestTimeout = 35 * time.Second
-)
-
-// ErrNoDrainSocket reports that this node has no drain socket for the volume,
-// which is how a node learns it does not serve it. It is deliberately distinct
-// from a socket that answered badly: only the former is worth routing onward,
-// and conflating the two makes a node re-run a drain that has already failed.
-var ErrNoDrainSocket = errors.New("no drain socket on this node")
-
-// DrainVolumeSocket asks the NBD plugin serving volumeID on this node to flush
-// its WAL chunks and live checkpoint to S3, then waits for the ack. The plugin
-// creates the socket under {dataDir}/viperblock/{volumeID}/, so this only
-// succeeds on the node hosting the volume; the daemon calls it when a drain
-// command is routed to it.
-func DrainVolumeSocket(dataDir, volumeID string) error {
-	sockPath := filepath.Join(dataDir, "viperblock", volumeID, "snapshot.sock")
-	conn, err := net.DialTimeout("unix", sockPath, drainDialTimeout)
-	if err != nil {
-		return fmt.Errorf("%w: dial %s: %w", ErrNoDrainSocket, sockPath, err)
-	}
-	defer conn.Close()
-
-	_ = conn.SetDeadline(time.Now().Add(drainAckTimeout))
-	buf := make([]byte, 8)
-	n, err := conn.Read(buf)
-	if err != nil {
-		return fmt.Errorf("read drain ack for %s: %w", volumeID, err)
-	}
-	if !strings.HasPrefix(string(buf[:n]), "OK") {
-		return fmt.Errorf("drain of %s did not ack OK: %q", volumeID, string(buf[:n]))
-	}
-	return nil
-}
-
-// drainVolume flushes the volume's in-flight writes to S3 before the snapshot
-// reads the live checkpoint from there.
-func (s *SnapshotServiceImpl) drainVolume(ctx context.Context, volumeID string, fallbackState, fallbackInstance string, accountID string) error {
-	return DrainVolume(ctx, s.config, s.store, s.natsConn, volumeID, fallbackState, fallbackInstance, accountID)
-}
-
-// DrainVolume flushes the volume's in-flight writes to S3 before a live
-// checkpoint is read from there. Shared by ec2.CreateSnapshot and
-// ec2.CreateImage's running-instance path (domains/ec2/image) so the two
-// live-snapshot call sites cannot diverge on when a drain is required.
-//
-// The drain socket lives on the node serving the volume, but both callers are
-// queue-grouped across every node, so the node answering the request usually
-// is not that node. Whether a drain is required is therefore decided from the
-// volume's attachment record, never from whether the socket happens to be
-// local: an attached volume is being written to right now, and reading its
-// checkpoint without draining silently captures an older one.
-func DrainVolume(ctx context.Context, cfg *config.Config, store objectstore.ObjectStore, natsConn *nats.Conn, volumeID string, fallbackState, fallbackInstance string, accountID string) error {
-	// A metadata-only snapshot never reads the live checkpoint, so there is
-	// nothing for a drain to make current.
-	if cfg == nil || cfg.Predastore.Host == "" {
-		return nil
-	}
-
-	state, instanceID, err := volumeAttachment(ctx, store, cfg.Predastore.Bucket, volumeID, fallbackState, fallbackInstance)
-	if err != nil {
-		return err
-	}
-
-	// Nothing is writing to an unattached volume: the checkpoint its Close()
-	// left behind is the current one. Decide this before touching the socket —
-	// dialing it is not a probe, it makes the plugin run a full flush.
-	if state != "in-use" || instanceID == "" {
-		slog.InfoContext(ctx, "DrainVolume: volume not attached, snapshotting the checkpoint left by Close (stopped instance path)",
-			"volumeId", volumeID, "state", state, "attachedInstance", instanceID)
-		return nil
-	}
-
-	// Fast path: the volume is served by this node, so no hop is needed. This is
-	// every single-node deployment, and the node hosting the instance on a
-	// cluster.
-	localErr := DrainVolumeSocket(cfg.DataDir, volumeID)
-	if localErr == nil {
-		return nil
-	}
-
-	// A socket that answered but did not ack means this node does serve the
-	// volume and the flush itself failed. Routing that onward would land back on
-	// this same node and repeat the drain that has already failed.
-	if !errors.Is(localErr, ErrNoDrainSocket) {
-		return fmt.Errorf("drain volume %s on this node: %w", volumeID, localErr)
-	}
-
-	slog.InfoContext(ctx, "DrainVolume: volume attached, routing drain to the node hosting it",
-		"volumeId", volumeID, "instanceId", instanceID)
-	return drainOnHostNode(ctx, natsConn, volumeID, instanceID, accountID)
-}
-
-// volumeAttachment returns the volume's authoritative state and attached
-// instance. state.json is control-plane-owned and authoritative; the fallback
-// pair (passed in from the caller's own volume record) is rewritten by the
-// live NBD plugin from its stale in-memory state and is only used for volumes
-// predating the split.
-func volumeAttachment(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string, fallbackState, fallbackInstance string) (state, instanceID string, err error) {
-	rec, found, err := volumestate.Read(ctx, store, bucket, volumeID)
-	if err != nil {
-		return "", "", fmt.Errorf("read volume state for %s: %w", volumeID, err)
-	}
-	if found {
-		return rec.State, rec.AttachedInstance, nil
-	}
-	return fallbackState, fallbackInstance, nil
-}
-
-// drainOnHostNode issues the drain on the node hosting instanceID. Only that
-// node subscribes ec2.cmd.{instanceID}, so the command is self-routing and no
-// volume-to-node resolution is needed here.
-//
-// An attachment record is not proof of a live writer: stop deliberately leaves
-// a boot volume attached (runtime/daemon/vm_adapters.go's volumeMounterAdapter.Unmount)
-// while tearing down both the NBD plugin and this subscription. The two
-// not-running signals below are therefore the stopped-instance path, not a
-// failure — treating them as one would make every stopped instance's root
-// volume permanently unsnapshottable.
-func drainOnHostNode(ctx context.Context, natsConn *nats.Conn, volumeID, instanceID, accountID string) error {
-	command := ec2v1.EC2InstanceCommand{
-		ID:              instanceID,
-		Attributes:      ec2v1.EC2CommandAttributes{DrainVolume: true},
-		DrainVolumeData: &ec2v1.DrainVolumeData{VolumeID: volumeID},
-	}
-
-	resp, err := natsmsg.NATSRequest[ec2v1.DrainVolumeResponse](ctx, natsConn,
-		ec2v1.InstanceCommandSubject(instanceID), command, drainRequestTimeout, accountID)
-	if err != nil {
-		// No subscriber at all: the instance runs nowhere in the cluster, so it
-		// was stopped (stop migrates it to shared KV before unsubscribing) and
-		// its volumes were sealed by Close. Warn rather than Info because the
-		// same shape appears if the hosting node has lost NATS while its VM
-		// keeps writing, which yields a stale snapshot as it did before routing
-		// existed. A live host that answers and fails is still a hard error.
-		if errors.Is(err, nats.ErrNoResponders) {
-			slog.WarnContext(ctx, "drainVolume: no node hosts the instance, treating the volume as stopped and snapshotting its sealed checkpoint",
-				"volumeId", volumeID, "instanceId", instanceID)
-			return nil
-		}
-		return fmt.Errorf("drain volume %s on the node hosting %s: %w", volumeID, instanceID, err)
-	}
-
-	switch resp.Status {
-	case ec2v1.DrainVolumeStatusDrained:
-		return nil
-	// The host still holds the instance but it is not running: nothing is
-	// writing, so there is nothing to flush. This is a host-drain stop, which
-	// keeps the VM (and this subscription) in place after unmounting.
-	case ec2v1.DrainVolumeStatusNotRunning:
-		slog.InfoContext(ctx, "drainVolume: instance is not running on its host, snapshotting the volume's sealed checkpoint",
-			"volumeId", volumeID, "instanceId", instanceID)
-		return nil
-	default:
-		return fmt.Errorf("drain volume %s on the node hosting %s: unexpected ack %q", volumeID, instanceID, resp.Status)
-	}
-}
-
 // describeSnapshotsValidFilters defines the set of filter names accepted by DescribeSnapshots.
 var describeSnapshotsValidFilters = map[string]bool{
 	"snapshot-id": true,
@@ -740,6 +557,21 @@ func unreadableRecordError(ctx context.Context, snapshotID, accountID, kind stri
 		"Cannot confirm that snapshot '%s' is not in use: %d %s record(s) could not be read", snapshotID, len(unreadable), kind)
 }
 
+// ProviderCreateError maps a provider CreateSnapshot failure of volumeID to
+// the code a client can act on: retry-later for a provider that cannot answer
+// now, IncorrectState for one that needs the volume detached, else internal.
+func ProviderCreateError(volumeID string, err error) error {
+	switch {
+	case errors.Is(err, ebsprovider.ErrUnavailable):
+		return errors.New(awserrors.ErrorServiceUnavailable)
+	case errors.Is(err, ebsprovider.ErrVolumeInUse):
+		return awserrors.Errorf(awserrors.ErrorIncorrectState,
+			"The volume '%s' is in use. Detach it before creating a snapshot.", volumeID)
+	default:
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+}
+
 // providerDeleteError maps a provider DeleteSnapshot failure to the code a
 // client can act on. A transport failure is retryable: the delete is idempotent.
 func providerDeleteError(snapshotID string, err error) error {

spinifex/domains/ec2/snapshot/service_impl_test.go

@@ -1519,3 +1519,55 @@ func TestDropEmptySnapshotRefsDoesNotDropAConcurrentAdd(t *testing.T) {
 		assert.True(t, has, "iteration %d: the concurrent add was dropped by the empty-key delete", i)
 	}
 }
+
+// createFailingProvider fails only CreateSnapshot, with err.
+type createFailingProvider struct {
+	ebsprovider.EBSProvider
+
+	err error
+}
+
+func (p *createFailingProvider) CreateSnapshot(context.Context, ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	return nil, p.err
+}
+
+// A provider that cannot snapshot a volume while it is attached refuses it
+// as in use, which the client can act on by detaching the volume first.
+func TestCreateSnapshot_ProviderVolumeInUseIsIncorrectState(t *testing.T) {
+	svc, store := setupTestSnapshotService(t)
+	createTestVolume(t, svc, store, "vol-create-inuse", 10)
+	svc.SetEBSProvider(&createFailingProvider{EBSProvider: svc.provider,
+		err: &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "published"}})
+
+	_, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-create-inuse")}, testAccountID)
+
+	code, message := clientErrorDetail(t, err)
+	assert.Equal(t, awserrors.ErrorIncorrectState, code)
+	assert.Contains(t, message, "vol-create-inuse")
+	assert.Contains(t, message, "Detach")
+}
+
+// A provider that cannot answer now, such as one refusing a volume another
+// node holds, is retry-later for the client; any other failure is internal.
+func TestCreateSnapshot_ProviderFailureMapsToSpecificError(t *testing.T) {
+	cases := []struct {
+		name     string
+		err      error
+		wantCode string
+	}{
+		{"provider unavailable", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}, awserrors.ErrorServiceUnavailable},
+		{"internal failure", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: "boom"}, awserrors.ErrorServerInternal},
+		{"transport failure", errors.New("connection reset"), awserrors.ErrorServerInternal},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			svc, store := setupTestSnapshotService(t)
+			createTestVolume(t, svc, store, "vol-create-fail", 10)
+			svc.SetEBSProvider(&createFailingProvider{EBSProvider: svc.provider, err: tc.err})
+
+			_, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-create-fail")}, testAccountID)
+			require.Error(t, err)
+			assert.Equal(t, tc.wantCode, err.Error())
+		})
+	}
+}

spinifex/domains/ec2/volumestate/volumestate.go

@@ -1,78 +0,0 @@
-// Package volumestate reads and writes the control-plane-owned attachment
-// state for an EBS volume, persisted to a per-volume state.json object.
-//
-// It is kept out of config.json because config.json is rewritten by the live
-// nbdkit VB on every SaveState (clobbering any State the control plane wrote
-// there) and is a sealed object for encrypted volumes (a second writer reuses
-// the AES-GCM nonce). state.json is plaintext, viperblock never touches it, so
-// the control plane is its single writer. Readers outside the volume service
-// (the snapshot path) need the same record, which is why it lives here.
-package volumestate
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-)
-
-// Record is the control-plane-owned attachment state of a volume.
-type Record struct {
-	State            string    `json:"state"`
-	AttachedInstance string    `json:"attachedInstance"`
-	DeviceName       string    `json:"deviceName"`
-	AttachedAt       time.Time `json:"attachedAt"`
-}
-
-// Key is the S3 key for a volume's control-plane state object.
-func Key(volumeID string) string { return volumeID + "/state.json" }
-
-// Write persists the attachment state, replacing any previous record.
-func Write(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string, rec Record) error {
-	data, err := json.Marshal(rec)
-	if err != nil {
-		return fmt.Errorf("marshal volume state: %w", err)
-	}
-	_, err = store.PutObject(ctx, &s3.PutObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(Key(volumeID)),
-		Body:   bytes.NewReader(data),
-	})
-	if err != nil {
-		return fmt.Errorf("failed to write volume state to S3: %w", err)
-	}
-	return nil
-}
-
-// Read returns the attachment state. found=false with a nil error means the
-// object is absent (a volume predating the state.json split), in which case the
-// caller falls back to the State embedded in config.json.
-func Read(ctx context.Context, store objectstore.ObjectStore, bucket, volumeID string) (Record, bool, error) {
-	getResult, err := store.GetObject(ctx, &s3.GetObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(Key(volumeID)),
-	})
-	if err != nil {
-		if objectstore.IsNoSuchKeyError(err) {
-			return Record{}, false, nil
-		}
-		return Record{}, false, fmt.Errorf("failed to get volume state: %w", err)
-	}
-	defer getResult.Body.Close()
-
-	body, err := io.ReadAll(getResult.Body)
-	if err != nil {
-		return Record{}, false, fmt.Errorf("failed to read volume state body: %w", err)
-	}
-	var rec Record
-	if err := json.Unmarshal(body, &rec); err != nil {
-		return Record{}, false, fmt.Errorf("failed to unmarshal volume state: %w", err)
-	}
-	return rec, true, nil
-}

spinifex/domains/iam/service_impl.go

@@ -597,8 +597,14 @@ func (s *IAMServiceImpl) DeleteUser(accountID string, input *iam.DeleteUserInput
 	if len(user.AttachedPolicies) > 0 {
 		return nil, errDetachPoliciesFirst
 	}
-	if len(user.Groups) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+	for _, groupName := range user.Groups {
+		if _, err := s.getGroup(ctx, accountID, groupName); err == nil {
+			return nil, awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove users from group first.")
+		} else if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
+			return nil, fmt.Errorf("get group: %w", err)
+		}
+		slog.Warn("DeleteUser: ignoring group name with no group record",
+			"accountID", accountID, "userName", userName, "groupName", groupName)
 	}
 	if len(user.InlinePolicies) > 0 {
 		return nil, errDeletePoliciesFirst

spinifex/domains/iam/service_impl_test.go

@@ -3180,3 +3180,37 @@ func TestDeleteUser_RevocationFailureKeepsDelete(t *testing.T) {
 	_, err = svc.GetUser(testAccountID, &iam.GetUserInput{UserName: aws.String("leaver")})
 	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIAMNoSuchEntity))
 }
+
+func TestDeleteUser_DanglingGroupName(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	createTestUser(t, svc, "danglinggroup")
+	createTestGroup(t, svc, "gone")
+	_, err := svc.AddUserToGroup(testAccountID, &iam.AddUserToGroupInput{
+		UserName: aws.String("danglinggroup"), GroupName: aws.String("gone"),
+	})
+	require.NoError(t, err)
+	require.NoError(t, svc.groupsBucket.Delete(t.Context(), testAccountID+".gone"))
+
+	_, err = svc.DeleteUser(testAccountID, &iam.DeleteUserInput{UserName: aws.String("danglinggroup")})
+	require.NoError(t, err)
+
+	_, err = svc.GetUser(testAccountID, &iam.GetUserInput{UserName: aws.String("danglinggroup")})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), awserrors.ErrorIAMNoSuchEntity)
+}
+
+func TestDeleteUser_LiveGroupStillConflicts(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	createTestUser(t, svc, "ingroup")
+	createTestGroup(t, svc, "live")
+	_, err := svc.AddUserToGroup(testAccountID, &iam.AddUserToGroupInput{
+		UserName: aws.String("ingroup"), GroupName: aws.String("live"),
+	})
+	require.NoError(t, err)
+
+	_, err = svc.DeleteUser(testAccountID, &iam.DeleteUserInput{UserName: aws.String("ingroup")})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), awserrors.ErrorIAMDeleteConflict)
+}

spinifex/foundation/messaging/nats/nats_trace.go

@@ -48,10 +48,10 @@ func ExtractTraceContext(ctx context.Context, msg *nats.Msg) context.Context {
 // front door to whichever service ends up doing the work.
 const AttrAccountID = "aws.account_id"
 
-// accountAttrs returns the account attribute for accountID, or nothing. Work
+// AccountAttrs returns the account attribute for accountID, or nothing. Work
 // with no caller — a sweep, a reconciler, a node-to-node probe — is left
 // unattributed rather than being credited to whoever happens to be admin.
-func accountAttrs(accountID string) []attribute.KeyValue {
+func AccountAttrs(accountID string) []attribute.KeyValue {
 	if accountID == "" {
 		return nil
 	}
@@ -66,7 +66,7 @@ func StartConsumerSpan(msg *nats.Msg) (context.Context, trace.Span) {
 	attrs := append([]attribute.KeyValue{
 		attribute.String("messaging.system", "nats"),
 		attribute.String("messaging.destination.name", msg.Subject),
-	}, accountAttrs(AccountIDFromMsg(msg))...)
+	}, AccountAttrs(AccountIDFromMsg(msg))...)
 
 	return otel.Tracer(natsTracerName).Start(ctx, "NATS "+msg.Subject,
 		trace.WithSpanKind(trace.SpanKindConsumer),
@@ -78,7 +78,7 @@ func startProducerSpan(ctx context.Context, subject, accountID string) (context.
 	attrs := append([]attribute.KeyValue{
 		attribute.String("messaging.system", "nats"),
 		attribute.String("messaging.destination.name", subject),
-	}, accountAttrs(accountID)...)
+	}, AccountAttrs(accountID)...)
 
 	return otel.Tracer(natsTracerName).Start(ctx, "NATS "+subject,
 		trace.WithSpanKind(trace.SpanKindClient),

spinifex/operator/accountlifecycle/reapers_iam.go

@@ -162,6 +162,24 @@ func (r *iamUserReaper) Delete(_ context.Context, accountID string, resource Res
 		}
 	}
 
+	// AWS refuses DeleteUser while the user is in a group.
+	groups, err := r.svc.ListGroupsForUser(accountID, &iam.ListGroupsForUserInput{UserName: aws.String(resource.ID)})
+	if err != nil && !isAlreadyGone(err) {
+		return err
+	}
+	if groups != nil {
+		for _, group := range groups.Groups {
+			if group == nil || group.GroupName == nil {
+				continue
+			}
+			if _, err := r.svc.RemoveUserFromGroup(accountID, &iam.RemoveUserFromGroupInput{
+				UserName: aws.String(resource.ID), GroupName: group.GroupName,
+			}); err != nil && !isAlreadyGone(err) {
+				return err
+			}
+		}
+	}
+
 	_, err = r.svc.DeleteUser(accountID, &iam.DeleteUserInput{UserName: aws.String(resource.ID)})
 	return ignoreAlreadyGone(err)
 }
@@ -249,8 +267,46 @@ func (r *iamGroupReaper) List(_ context.Context, accountID string) ([]Resource,
 	return found, nil
 }
 
+// Delete strips the group's policies first: AWS refuses to delete a group that
+// still has any. Its members were removed with their users, earlier in the stage.
 func (r *iamGroupReaper) Delete(_ context.Context, accountID string, resource Resource, _ bool) error {
-	_, err := r.svc.DeleteGroup(accountID, &iam.DeleteGroupInput{GroupName: aws.String(resource.ID)})
+	attached, err := r.svc.ListAttachedGroupPolicies(accountID, &iam.ListAttachedGroupPoliciesInput{
+		GroupName: aws.String(resource.ID),
+	})
+	if err != nil && !isAlreadyGone(err) {
+		return err
+	}
+	if attached != nil {
+		for _, policy := range attached.AttachedPolicies {
+			if policy == nil || policy.PolicyArn == nil {
+				continue
+			}
+			if _, err := r.svc.DetachGroupPolicy(accountID, &iam.DetachGroupPolicyInput{
+				GroupName: aws.String(resource.ID), PolicyArn: policy.PolicyArn,
+			}); err != nil && !isAlreadyGone(err) {
+				return err
+			}
+		}
+	}
+
+	inline, err := r.svc.ListGroupPolicies(accountID, &iam.ListGroupPoliciesInput{GroupName: aws.String(resource.ID)})
+	if err != nil && !isAlreadyGone(err) {
+		return err
+	}
+	if inline != nil {
+		for _, name := range inline.PolicyNames {
+			if name == nil {
+				continue
+			}
+			if _, err := r.svc.DeleteGroupPolicy(accountID, &iam.DeleteGroupPolicyInput{
+				GroupName: aws.String(resource.ID), PolicyName: name,
+			}); err != nil && !isAlreadyGone(err) {
+				return err
+			}
+		}
+	}
+
+	_, err = r.svc.DeleteGroup(accountID, &iam.DeleteGroupInput{GroupName: aws.String(resource.ID)})
 	return ignoreAlreadyGone(err)
 }
 

spinifex/operator/accountlifecycle/reapers_iam_groups_test.go

@@ -0,0 +1,95 @@
+package accountlifecycle
+
+//test:in-package — drives the unexported identity reapers against the real IAM
+// service, so the DeleteConflict refusals are the real ones and not a fake's.
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const groupTestAccount = "000000000042"
+
+const groupTestPolicyDoc = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListAllMyBuckets","Resource":"*"}]}`
+
+func newRealIAM(t *testing.T) *iamdomain.IAMServiceImpl {
+	t.Helper()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	masterKey, err := iamdomain.GenerateMasterKey()
+	require.NoError(t, err)
+	svc, err := iamdomain.NewIAMServiceImpl(t.Context(), nc, masterKey)
+	require.NoError(t, err)
+	return svc
+}
+
+// drainIdentity runs the identity-stage reapers in registration order for a
+// few passes, as a stage does, and returns what each reaper still lists and
+// the delete errors of the last pass.
+func drainIdentity(t *testing.T, svc iamdomain.IAMService) (remaining []Resource, lastErrs []error) {
+	t.Helper()
+	for range 3 {
+		lastErrs = nil
+		for _, reaper := range IAMReapers(svc) {
+			if reaper.Stage() != StageIdentity {
+				continue
+			}
+			found, err := reaper.List(testCtx(t), groupTestAccount)
+			require.NoError(t, err)
+			for _, resource := range found {
+				if err := reaper.Delete(testCtx(t), groupTestAccount, resource, false); err != nil {
+					lastErrs = append(lastErrs, err)
+				}
+			}
+		}
+	}
+	for _, reaper := range IAMReapers(svc) {
+		if reaper.Stage() != StageIdentity {
+			continue
+		}
+		found, err := reaper.List(testCtx(t), groupTestAccount)
+		require.NoError(t, err)
+		remaining = append(remaining, found...)
+	}
+	return remaining, lastErrs
+}
+
+func seedGroupAccount(t *testing.T, svc *iamdomain.IAMServiceImpl) string {
+	t.Helper()
+	_, err := svc.CreateUser(groupTestAccount, &iam.CreateUserInput{UserName: aws.String("u")})
+	require.NoError(t, err)
+	_, err = svc.CreateGroup(groupTestAccount, &iam.CreateGroupInput{GroupName: aws.String("g")})
+	require.NoError(t, err)
+	_, err = svc.AddUserToGroup(groupTestAccount, &iam.AddUserToGroupInput{UserName: aws.String("u"), GroupName: aws.String("g")})
+	require.NoError(t, err)
+	policy, err := svc.CreatePolicy(groupTestAccount, &iam.CreatePolicyInput{
+		PolicyName: aws.String("p"), PolicyDocument: aws.String(groupTestPolicyDoc),
+	})
+	require.NoError(t, err)
+	_, err = svc.AttachGroupPolicy(groupTestAccount, &iam.AttachGroupPolicyInput{
+		GroupName: aws.String("g"), PolicyArn: policy.Policy.Arn,
+	})
+	require.NoError(t, err)
+	_, err = svc.PutGroupPolicy(groupTestAccount, &iam.PutGroupPolicyInput{
+		GroupName: aws.String("g"), PolicyName: aws.String("inline"), PolicyDocument: aws.String(groupTestPolicyDoc),
+	})
+	require.NoError(t, err)
+	return *policy.Policy.Arn
+}
+
+// A user in a group, a group carrying an attached and an inline policy, and the
+// customer policy attached to it all drain; none of them may strand the account.
+func TestIdentityStageDrainsGroupsMembersAndGroupPolicies(t *testing.T) {
+	svc := newRealIAM(t)
+	seedGroupAccount(t, svc)
+
+	remaining, lastErrs := drainIdentity(t, svc)
+
+	assert.Empty(t, remaining)
+	assert.Empty(t, lastErrs)
+}

spinifex/operator/accountlifecycle/reapers_iam_test.go

@@ -34,6 +34,9 @@ type fakeIAM struct {
 	profileRoles     map[string][]string
 	attachedUser     []string
 	inlineUser       []string
+	userGroups       []string
+	attachedGroup    []string
+	inlineGroup      []string
 	attachedRole     []string
 	inlineRole       []string
 
@@ -99,6 +102,48 @@ func (f *fakeIAM) DeleteAccessKey(_ string, _ *iam.DeleteAccessKeyInput) (*iam.D
 	return &iam.DeleteAccessKeyOutput{}, nil
 }
 
+func (f *fakeIAM) ListGroupsForUser(_ string, _ *iam.ListGroupsForUserInput) (*iam.ListGroupsForUserOutput, error) {
+	f.record("ListGroupsForUser")
+	out := &iam.ListGroupsForUserOutput{}
+	for _, name := range f.userGroups {
+		out.Groups = append(out.Groups, &iam.Group{GroupName: aws.String(name)})
+	}
+	return out, nil
+}
+
+func (f *fakeIAM) RemoveUserFromGroup(_ string, _ *iam.RemoveUserFromGroupInput) (*iam.RemoveUserFromGroupOutput, error) {
+	f.record("RemoveUserFromGroup")
+	return &iam.RemoveUserFromGroupOutput{}, nil
+}
+
+func (f *fakeIAM) ListAttachedGroupPolicies(_ string, _ *iam.ListAttachedGroupPoliciesInput) (*iam.ListAttachedGroupPoliciesOutput, error) {
+	f.record("ListAttachedGroupPolicies")
+	out := &iam.ListAttachedGroupPoliciesOutput{}
+	for _, arn := range f.attachedGroup {
+		out.AttachedPolicies = append(out.AttachedPolicies, &iam.AttachedPolicy{PolicyArn: aws.String(arn)})
+	}
+	return out, nil
+}
+
+func (f *fakeIAM) DetachGroupPolicy(_ string, _ *iam.DetachGroupPolicyInput) (*iam.DetachGroupPolicyOutput, error) {
+	f.record("DetachGroupPolicy")
+	return &iam.DetachGroupPolicyOutput{}, nil
+}
+
+func (f *fakeIAM) ListGroupPolicies(_ string, _ *iam.ListGroupPoliciesInput) (*iam.ListGroupPoliciesOutput, error) {
+	f.record("ListGroupPolicies")
+	out := &iam.ListGroupPoliciesOutput{}
+	for _, name := range f.inlineGroup {
+		out.PolicyNames = append(out.PolicyNames, aws.String(name))
+	}
+	return out, nil
+}
+
+func (f *fakeIAM) DeleteGroupPolicy(_ string, _ *iam.DeleteGroupPolicyInput) (*iam.DeleteGroupPolicyOutput, error) {
+	f.record("DeleteGroupPolicy")
+	return &iam.DeleteGroupPolicyOutput{}, nil
+}
+
 func (f *fakeIAM) ListAttachedUserPolicies(_ string, _ *iam.ListAttachedUserPoliciesInput) (*iam.ListAttachedUserPoliciesOutput, error) {
 	f.record("ListAttachedUserPolicies")
 	out := &iam.ListAttachedUserPoliciesOutput{}
@@ -337,10 +382,41 @@ func TestUserReaperStripsPoliciesBeforeDeleting(t *testing.T) {
 	assert.Equal(t, []string{
 		"ListAttachedUserPolicies", "DetachUserPolicy",
 		"ListUserPolicies", "DeleteUserPolicy",
+		"ListGroupsForUser", "DeleteUser",
+	}, svc.calls)
+}
+
+func TestUserReaperRemovesGroupMembershipsBeforeDeleting(t *testing.T) {
+	svc := newFakeIAM()
+	svc.userGroups = []string{"g1", "g2"}
+
+	err := (&iamUserReaper{svc: svc}).Delete(testCtx(t), "000000000042",
+		Resource{Kind: "iam-user", ID: "admin"}, false)
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{
+		"ListAttachedUserPolicies", "ListUserPolicies",
+		"ListGroupsForUser", "RemoveUserFromGroup", "RemoveUserFromGroup",
 		"DeleteUser",
 	}, svc.calls)
 }
 
+func TestGroupReaperStripsPoliciesBeforeDeleting(t *testing.T) {
+	svc := newFakeIAM()
+	svc.attachedGroup = []string{"arn:aws:iam::000000000042:policy/p"}
+	svc.inlineGroup = []string{"inline-1"}
+
+	err := (&iamGroupReaper{svc: svc}).Delete(testCtx(t), "000000000042",
+		Resource{Kind: "iam-group", ID: "g"}, false)
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{
+		"ListAttachedGroupPolicies", "DetachGroupPolicy",
+		"ListGroupPolicies", "DeleteGroupPolicy",
+		"DeleteGroup",
+	}, svc.calls)
+}
+
 func TestRoleReaperStripsPoliciesBeforeDeleting(t *testing.T) {
 	svc := newFakeIAM()
 	svc.attachedRole = []string{"arn:aws:iam::000000000042:policy/task"}

spinifex/providers/ebs/conformance/conformance.go

@@ -145,6 +145,16 @@ type SuiteConfig struct {
 	// the provider, standing in for a crash. Nil skips the dead-export case,
 	// for a provider whose export is not a separate process.
 	KillExport func(t *testing.T, published *ebsprovider.PublishedVolume)
+
+	// QueueOnOtherNode returns a client answered only on the queue subjects, by
+	// a node other than NodeID: what a request meets when the volume's owner
+	// does not answer. Nil skips the cases that need it.
+	QueueOnOtherNode func(t *testing.T) ebsprovider.EBSProvider
+
+	// RevokeExport makes the provider revoke a live export as it does when the
+	// volume is lost to another node, and returns the FenceNotice it sent; see
+	// CaptureFenceNotice. Nil skips the case, for a provider that never revokes.
+	RevokeExport func(t *testing.T, published *ebsprovider.PublishedVolume) ebsprovider.FenceNotice
 }
 
 func (c SuiteConfig) node() string {
@@ -210,9 +220,9 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 
 	t.Run("ContractVersions", func(t *testing.T) {
 		t.Run("listed versions are ascending and include the legacy version", func(t *testing.T) {
-			require.NotEmpty(t, contractVersions, "a caller cannot choose a version for a provider that lists none")
+			require.NotEmpty(t, contractVersions, "a provider must list the contract versions it serves")
 			assert.Contains(t, contractVersions, ebsprovider.LegacySchemaVersion,
-				"dropping the legacy version strands every caller still in an N/N-1 rollout")
+				"callers send version-neutral requests at the legacy version, so a provider must serve it")
 			for i := 1; i < len(contractVersions); i++ {
 				assert.Lessf(t, contractVersions[i-1], contractVersions[i], "contract versions %v are not strictly ascending", contractVersions)
 			}
@@ -630,6 +640,39 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			_, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned()})
 			require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
 		})
+
+		// Only the owner can drain what it buffers, so a node answering in its
+		// place must refuse rather than snapshot a checkpoint the owner has not
+		// flushed. The owner's own snapshot then succeeds.
+		t.Run("refused on the queue subject while another node holds the volume", func(t *testing.T) {
+			if !capabilities.OwnerRouting || !capabilities.Exclusion.SingleWriter() {
+				t.Skip("no owner routing behind cluster exclusion, so no node answers in place of an owner")
+			}
+			if cfg.QueueOnOtherNode == nil {
+				t.Skip("no queue-only client on another node configured")
+			}
+			provider := newProvider(t)
+			queue := newTrackingProvider(t, cfg.QueueOnOtherNode(t))
+			ctx := context.Background()
+			volumeID := cfg.id("vol-snap-held")
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			_, err = provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+
+			for _, version := range contractVersions {
+				snapshotID := cfg.id(fmt.Sprintf("snap-held-queue-v%d", version))
+				_, err = queue.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.Versioned{SchemaVersion: version}, SnapshotID: snapshotID, VolumeID: volumeID})
+				require.ErrorIs(t, err, ebsprovider.ErrUnavailable, "v%d: a node not holding the volume must refuse as retry-later", version)
+				if capabilities.SnapshotEnumeration {
+					assert.NotContains(t, drainSnapshotIDs(t, provider, 0), snapshotID)
+				}
+			}
+
+			snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: cfg.id("snap-held-owner"), VolumeID: volumeID})
+			require.NoError(t, err, "the holder answers for its own volume")
+			assert.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State)
+		})
 	})
 
 	t.Run("DeleteSnapshot", func(t *testing.T) {
@@ -950,6 +993,24 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			nbdInfoRetry(t, dialURI(t, publishedNBDURI(t, second)))
 		})
 
+		t.Run("revoking a live export sends a fence notice to its node", func(t *testing.T) {
+			if cfg.RevokeExport == nil {
+				t.Skip("this provider never revokes a live export")
+			}
+			provider := newProvider(t)
+			ctx := context.Background()
+			volumeID := cfg.id("vol-pub-revoked")
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			published, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+
+			notice := cfg.RevokeExport(t, published)
+			assert.Equal(t, volumeID, notice.VolumeID, "the daemon stops the guest on the volume the notice names")
+			assert.Equal(t, published.NodeID, notice.NodeID, "the notice must name the node whose export was revoked")
+			assert.True(t, ebsprovider.SupportsSchemaVersion(notice.SchemaVersion), "a daemon refuses a notice in a version it cannot read")
+		})
+
 		t.Run("not_found on absent volume", func(t *testing.T) {
 			provider := newProvider(t)
 			_, err := provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed"), NodeID: cfg.node()})

spinifex/providers/ebs/conformance/conformance_test.go

@@ -75,6 +75,28 @@ func TestMemoryProviderNamePrefixConformance(t *testing.T) {
 	}
 }
 
+// TestMemoryProviderFenceNoticeConformance runs the suite with the revocation
+// case enabled, so the contract's fence notice is held to the same check here
+// as on viperblockd.
+func TestMemoryProviderFenceNoticeConformance(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	shared := ebsprovider.NewMemoryProvider(conformance.ReferenceCapabilities)
+	shared.NotifyFencesOn(conn)
+
+	conformance.RunSuiteWithConfig(t, func(t *testing.T) ebsprovider.EBSProvider {
+		t.Helper()
+		return shared
+	}, conformance.SuiteConfig{
+		RevokeExport: func(t *testing.T, published *ebsprovider.PublishedVolume) ebsprovider.FenceNotice {
+			t.Helper()
+			return conformance.CaptureFenceNotice(t, conn, published.NodeID, func() error {
+				_, err := shared.RevokePublication(t.Context(), published.VolumeID, "node-elsewhere")
+				return err
+			})
+		},
+	})
+}
+
 // TestNATSProviderConformance runs the suite against a NATSProvider backed
 // by natsserve.Serve, the production-shaped neutral server, rather than a
 // test-only twin: this exercises the transport this package's implementors actually run in production, not a hand-written stand-in for it.

spinifex/providers/ebs/conformance/fence_notice.go

@@ -0,0 +1,60 @@
+package conformance
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/require"
+)
+
+// fenceNoticeWait bounds how long CaptureFenceNotice waits for the notice.
+const fenceNoticeWait = 10 * time.Second
+
+// CaptureFenceNotice answers nodeID's fence notices on conn as a daemon with no
+// guest on the volume does, starts revoke, and returns the first notice heard.
+// revoke runs on its own goroutine, since a sender may retry until answered.
+func CaptureFenceNotice(t *testing.T, conn *nats.Conn, nodeID string, revoke func() error) ebsprovider.FenceNotice {
+	t.Helper()
+	subject, err := ebsprovider.FenceNoticeSubject(nodeID)
+	require.NoError(t, err)
+
+	heard := make(chan ebsprovider.FenceNotice, 1)
+	sub, err := conn.Subscribe(subject, func(msg *nats.Msg) {
+		var notice ebsprovider.FenceNotice
+		ack := ebsprovider.FenceNoticeAck{Versioned: ebsprovider.NewVersioned()}
+		if err := json.Unmarshal(msg.Data, &notice); err != nil {
+			ack.Error = &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInvalidArgument, Message: err.Error()}
+		} else {
+			ack.Versioned = ebsprovider.ReplyVersioned(notice.SchemaVersion)
+			select {
+			case heard <- notice:
+			default:
+			}
+		}
+		if payload, err := json.Marshal(ack); err == nil {
+			_ = msg.Respond(payload)
+		}
+	})
+	require.NoError(t, err)
+	defer func() { _ = sub.Unsubscribe() }()
+	require.NoError(t, conn.Flush())
+
+	revoked := make(chan error, 1)
+	go func() { revoked <- revoke() }()
+
+	timeout := time.After(fenceNoticeWait)
+	for {
+		select {
+		case notice := <-heard:
+			return notice
+		case err := <-revoked:
+			require.NoError(t, err, "revoking the export")
+			revoked = nil
+		case <-timeout:
+			t.Fatalf("no fence notice on %s within %s of revoking the export; the daemon would leave its guest running on a revoked disk", subject, fenceNoticeWait)
+		}
+	}
+}

spinifex/providers/ebs/fence_notice.go

@@ -0,0 +1,53 @@
+package ebsprovider
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/nats-io/nats.go"
+)
+
+// NotifyFenced sends notice once and returns the instance the daemon stopped.
+// An ErrInvalidArgument error is terminal: the notice can never be sent. A
+// *ProviderError (internal for an undecodable ack) was heard; others were not.
+func NotifyFenced(ctx context.Context, conn *nats.Conn, notice FenceNotice, timeout time.Duration) (string, error) {
+	if notice.VolumeID == "" {
+		return "", fmt.Errorf("%w: fence notice names no volume", ErrInvalidArgument)
+	}
+	subject, err := FenceNoticeSubject(notice.NodeID)
+	if err != nil {
+		return "", err
+	}
+	if conn == nil {
+		return "", fmt.Errorf("%w: no connection to send the fence notice on", ErrInvalidArgument)
+	}
+
+	reply, err := requestReply(ctx, conn, subject, timeout, notice)
+	if err != nil {
+		return "", err
+	}
+	var ack FenceNoticeAck
+	if err := json.Unmarshal(reply, &ack); err != nil {
+		// The daemon answered, so repeating the notice would only repeat this.
+		return "", &ProviderError{Code: ErrorCodeInternal, Message: fmt.Sprintf("decode %s ack: %v", subject, err)}
+	}
+	if ack.Error != nil {
+		return ack.InstanceID, ack.Error
+	}
+	return ack.InstanceID, nil
+}
+
+// FenceNoticeRetryable reports whether a NotifyFenced error leaves the notice
+// to be sent again: it was not heard, or the daemon refused it as unavailable.
+func FenceNoticeRetryable(err error) bool {
+	if err == nil || errors.Is(err, ErrInvalidArgument) {
+		return false
+	}
+	if _, refused := errors.AsType[*ProviderError](err); refused {
+		return errors.Is(err, ErrUnavailable)
+	}
+	return true
+}

spinifex/providers/ebs/fence_notice_test.go

@@ -0,0 +1,205 @@
+package ebsprovider
+
+import (
+	"context"
+	"encoding/json"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// answerFenceNotices replies to node-a's fence notices with reply.
+func answerFenceNotices(t *testing.T, conn *nats.Conn, reply []byte) {
+	t.Helper()
+	subject, err := FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	sub, err := conn.Subscribe(subject, func(msg *nats.Msg) { _ = msg.Respond(reply) })
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, conn.Flush())
+}
+
+func fenceNoticeFor(volumeID string) FenceNotice {
+	return FenceNotice{Versioned: NewVersioned(), VolumeID: volumeID, NodeID: "node-a", Winner: "node-b"}
+}
+
+// TestNotifyFenced_SeparatesHeardFromUnheard pins what a sender's retry turns
+// on: an answer of any kind ends the notice, and only silence repeats it.
+func TestNotifyFenced_SeparatesHeardFromUnheard(t *testing.T) {
+	t.Run("acknowledged", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned(), InstanceID: "i-stopped"})
+		require.NoError(t, err)
+		answerFenceNotices(t, conn, ack)
+
+		instanceID, err := NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		require.NoError(t, err)
+		assert.Equal(t, "i-stopped", instanceID)
+	})
+
+	t.Run("refused", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned(), Error: &ProviderError{Code: ErrorCodeInvalidArgument, Message: "no"}})
+		require.NoError(t, err)
+		answerFenceNotices(t, conn, ack)
+
+		_, err = NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		var refused *ProviderError
+		require.ErrorAs(t, err, &refused, "a refusal was heard, and repeating it would only be refused again")
+		assert.ErrorIs(t, err, ErrInvalidArgument)
+		assert.False(t, FenceNoticeRetryable(err))
+	})
+
+	t.Run("refused as unavailable", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned(), Error: &ProviderError{Code: ErrorCodeUnavailable, Message: "later"}})
+		require.NoError(t, err)
+		answerFenceNotices(t, conn, ack)
+
+		_, err = NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		require.ErrorIs(t, err, ErrUnavailable)
+		assert.True(t, FenceNoticeRetryable(err), "a daemon that cannot act yet may act on the next notice")
+	})
+
+	t.Run("undecodable answer", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		answerFenceNotices(t, conn, []byte("{"))
+
+		_, err := NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		var refused *ProviderError
+		require.ErrorAs(t, err, &refused, "the daemon answered, so the notice was heard")
+		assert.Equal(t, ErrorCodeInternal, refused.Code)
+		assert.False(t, FenceNoticeRetryable(err))
+	})
+
+	t.Run("nobody listening", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		_, err := NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		require.Error(t, err)
+		var refused *ProviderError
+		assert.NotErrorAs(t, err, &refused, "an unheard notice must read as unheard, or the sender stops retrying")
+		assert.True(t, FenceNoticeRetryable(err))
+	})
+
+	t.Run("connection closed", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		conn.Close()
+		_, err := NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		require.ErrorIs(t, err, nats.ErrConnectionClosed)
+		assert.True(t, FenceNoticeRetryable(err), "a connection that comes back can still carry the notice")
+	})
+}
+
+// TestNotifyFenced_AttributesTheNoticeToItsAccount holds the notice to the
+// contract's request path: the daemon's span names the account it acts for.
+func TestNotifyFenced_AttributesTheNoticeToItsAccount(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	subject, err := FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	accounts := make(chan string, 1)
+	ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned()})
+	require.NoError(t, err)
+	sub, err := conn.Subscribe(subject, func(msg *nats.Msg) {
+		accounts <- msg.Header.Get(AccountIDHeader)
+		_ = msg.Respond(ack)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, conn.Flush())
+
+	_, err = NotifyFenced(WithAccountID(t.Context(), "123456789012"), conn, fenceNoticeFor("vol-1"), time.Second)
+	require.NoError(t, err)
+	assert.Equal(t, "123456789012", <-accounts)
+}
+
+// TestNotifyFenced_RefusesAnUnaddressableNotice keeps a notice that can never
+// be sent from reading as unheard, which a retrying sender would repeat forever.
+func TestNotifyFenced_RefusesAnUnaddressableNotice(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	cases := map[string]struct {
+		conn   *nats.Conn
+		notice FenceNotice
+	}{
+		"no volume": {conn, FenceNotice{Versioned: NewVersioned(), NodeID: "node-a"}},
+		"no node":   {conn, FenceNotice{Versioned: NewVersioned(), VolumeID: "vol-1"}},
+		"bad node":  {conn, FenceNotice{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node.a"}},
+		"no conn":   {nil, fenceNoticeFor("vol-1")},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			_, err := NotifyFenced(context.Background(), tc.conn, tc.notice, time.Second)
+			require.ErrorIs(t, err, ErrInvalidArgument)
+			assert.False(t, FenceNoticeRetryable(err), "a notice that can never be sent must not be retried")
+		})
+	}
+}
+
+// TestMemoryProviderRevokePublication_RetriesAnUnheardNotice keeps the memory
+// provider to the fence's delivery rule: a notice nobody answered is sent
+// again until one does, not dropped after the first try.
+func TestMemoryProviderRevokePublication_RetriesAnUnheardNotice(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	ctx := t.Context()
+	provider := NewMemoryProvider(Capabilities{})
+	provider.fenceNoticeTimeout = 100 * time.Millisecond
+	provider.fenceNoticeEvery = 10 * time.Millisecond
+	provider.NotifyFencesOn(conn)
+	_, err := provider.CreateVolume(ctx, CreateVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", CapacityRange: CapacityRange{RequiredBytes: 1 << 30}})
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-a"})
+	require.NoError(t, err)
+
+	subject, err := FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned(), InstanceID: "i-stopped"})
+	require.NoError(t, err)
+	var heard atomic.Int32
+	sub, err := conn.Subscribe(subject, func(msg *nats.Msg) {
+		if heard.Add(1) == 1 {
+			return // the first notice is lost, as on a daemon still reconnecting
+		}
+		_ = msg.Respond(ack)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, conn.Flush())
+
+	stopped, err := provider.RevokePublication(ctx, "vol-1", "node-b")
+	require.NoError(t, err, "a lost first notice must be sent again")
+	assert.Equal(t, "i-stopped", stopped)
+	assert.Equal(t, int32(2), heard.Load())
+}
+
+func TestMemoryProviderRevokePublication(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	ctx := context.Background()
+	provider := NewMemoryProvider(Capabilities{})
+	_, err := provider.CreateVolume(ctx, CreateVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", CapacityRange: CapacityRange{RequiredBytes: 1 << 30}})
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-a"})
+	require.NoError(t, err)
+
+	_, err = provider.RevokePublication(ctx, "vol-1", "node-b")
+	require.ErrorIs(t, err, ErrUnavailable, "with nowhere to announce it, a revocation would leave the guest unstopped")
+	stillPublished, err := provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-b"})
+	require.ErrorIs(t, err, ErrVolumeInUse, "a refused revocation must leave the publication in place")
+	assert.Nil(t, stillPublished)
+
+	ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned()})
+	require.NoError(t, err)
+	answerFenceNotices(t, conn, ack)
+	provider.NotifyFencesOn(conn)
+
+	_, err = provider.RevokePublication(ctx, "vol-1", "node-b")
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-b"})
+	require.NoError(t, err, "a revoked publication no longer holds the volume")
+
+	_, err = provider.RevokePublication(ctx, "vol-never", "node-b")
+	require.ErrorIs(t, err, ErrNotFound)
+}

spinifex/providers/ebs/memory.go

@@ -2,12 +2,15 @@ package ebsprovider
 
 import (
 	"bytes"
+	"cmp"
 	"context"
 	"fmt"
 	"maps"
 	"slices"
 	"sync"
 	"time"
+
+	"github.com/nats-io/nats.go"
 )
 
 // MemoryProvider is a deterministic, concurrency-safe Provider used by
@@ -19,6 +22,10 @@ type MemoryProvider struct {
 	volumes      map[string]*memoryVolume
 	snapshots    map[string]*Snapshot
 	now          func() time.Time
+	fenceConn    *nats.Conn
+	// Zero takes the defaults; tests shorten them.
+	fenceNoticeTimeout time.Duration
+	fenceNoticeEvery   time.Duration
 }
 
 type memoryVolume struct {
@@ -406,13 +413,73 @@ func (m *MemoryProvider) UnpublishVolume(_ context.Context, req UnpublishVolumeR
 	if volume == nil || volume.published == nil || volume.published.NodeID != req.NodeID {
 		return result, nil
 	}
-	volume.published = nil
-	volume.readOnly = false
-	volume.volume.State = VolumeStateAvailable
+	volume.unpublish()
 	result.TornDown = req.SchemaVersion >= NoSealSchemaVersion
 	return result, nil
 }
 
+// NotifyFencesOn sets the connection RevokePublication announces fences on.
+func (m *MemoryProvider) NotifyFencesOn(conn *nats.Conn) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.fenceConn = conn
+}
+
+// unpublish drops the volume's publication. The caller holds the provider lock.
+func (v *memoryVolume) unpublish() {
+	v.published = nil
+	v.readOnly = false
+	v.volume.State = VolumeStateAvailable
+}
+
+// Defaults for RevokePublication's notice: each attempt's wait for an answer,
+// and the pause before an unanswered notice is sent again.
+const (
+	memoryFenceNoticeTimeout = 2 * time.Second
+	memoryFenceNoticeEvery   = time.Second
+)
+
+// RevokePublication tears volumeID's publication down as a provider that lost
+// the volume to winner does, then sends a FenceNotice to the node it was
+// published on until heard or ctx ends. It returns the instance stopped there.
+func (m *MemoryProvider) RevokePublication(ctx context.Context, volumeID, winner string) (string, error) {
+	m.mu.Lock()
+	conn := m.fenceConn
+	volume := m.volumes[volumeID]
+	timeout := cmp.Or(m.fenceNoticeTimeout, memoryFenceNoticeTimeout)
+	every := cmp.Or(m.fenceNoticeEvery, memoryFenceNoticeEvery)
+	if conn == nil {
+		m.mu.Unlock()
+		return "", fmt.Errorf("%w: no connection to announce the fence on", ErrUnavailable)
+	}
+	if volume == nil || volume.published == nil {
+		m.mu.Unlock()
+		return "", fmt.Errorf("%w: volume %s is not published", ErrNotFound, volumeID)
+	}
+	nodeID := volume.published.NodeID
+	volume.unpublish()
+	m.mu.Unlock()
+
+	notice := FenceNotice{
+		Versioned: NewVersioned(),
+		VolumeID:  volumeID,
+		NodeID:    nodeID,
+		Winner:    winner,
+		Reason:    "volume revoked in favour of " + winner,
+	}
+	for {
+		instanceID, err := NotifyFenced(ctx, conn, notice, timeout)
+		if !FenceNoticeRetryable(err) || conn.IsClosed() {
+			return instanceID, err
+		}
+		select {
+		case <-ctx.Done():
+			return "", fmt.Errorf("fence notice for %s never acknowledged: %w: %w", volumeID, ctx.Err(), err)
+		case <-time.After(every):
+		}
+	}
+}
+
 // HoldsUnsealedWrites answers that none are held: every write lands in this
 // process's map, which is the backend, before it is acknowledged.
 func (m *MemoryProvider) HoldsUnsealedWrites(_ context.Context, nodeID string, volumeIDs []string) ([]string, error) {

spinifex/providers/ebs/nats.go

@@ -35,8 +35,9 @@ const (
 const defaultRequestTimeout = 30 * time.Second
 
 // dataMovementTimeout bounds the subjects whose duration is a function of the
-// volume rather than of the control plane. Matches unmountSealTimeout, this
-// codebase's existing budget for an EBS operation the caller cannot bound.
+// volume rather than of the control plane. It must exceed viperblockd's 120s
+// kill grace plus a seal allowance, or an unpublish that uses the whole grace
+// could never be seen to succeed.
 const dataMovementTimeout = 180 * time.Second
 
 // dataMovementVerbs names every operation that references or moves real data,
@@ -88,11 +89,10 @@ func SnapshotCompletionSubject(snapshotID string) (string, error) {
 
 // Last tokens of the per-node subjects: ebs.provider.v1.<node>.<verb>.
 const (
-	publishVerb          = "mount"
-	unpublishVerb        = "unmount"
-	unsealedVerb         = "unsealed"
-	nodeCapabilitiesVerb = "capabilities"
-	fenceNoticeVerb      = "fenced"
+	publishVerb     = "mount"
+	unpublishVerb   = "unmount"
+	unsealedVerb    = "unsealed"
+	fenceNoticeVerb = "fenced"
 )
 
 // nodeSubjectWildcard is the subject matching verb on every node, for a server
@@ -101,10 +101,9 @@ func nodeSubjectWildcard(verb string) string { return subjectPrefix + "*." + ver
 
 // Wildcards over every node for each per-node subject.
 var (
-	PublishSubjectWildcard          = nodeSubjectWildcard(publishVerb)
-	UnpublishSubjectWildcard        = nodeSubjectWildcard(unpublishVerb)
-	UnsealedWritesSubjectWildcard   = nodeSubjectWildcard(unsealedVerb)
-	NodeCapabilitiesSubjectWildcard = nodeSubjectWildcard(nodeCapabilitiesVerb)
+	PublishSubjectWildcard        = nodeSubjectWildcard(publishVerb)
+	UnpublishSubjectWildcard      = nodeSubjectWildcard(unpublishVerb)
+	UnsealedWritesSubjectWildcard = nodeSubjectWildcard(unsealedVerb)
 )
 
 func nodeSubject(nodeID, verb string) (string, error) {
@@ -133,13 +132,6 @@ func UnsealedWritesSubject(nodeID string) (string, error) {
 	return nodeSubject(nodeID, unsealedVerb)
 }
 
-// NodeCapabilitiesSubject returns nodeID's per-node GetCapabilities subject
-// (ebs.provider.v1.<node>.capabilities). Unlike CapabilitiesSubject, which
-// any node's provider answers, only nodeID's provider answers it.
-func NodeCapabilitiesSubject(nodeID string) (string, error) {
-	return nodeSubject(nodeID, nodeCapabilitiesVerb)
-}
-
 // FenceNoticeSubject returns nodeID's fence-notice subject
 // (ebs.provider.v1.<node>.fenced), on which a provider that revoked a live
 // export there sends a FenceNotice for that node's daemon to acknowledge.
@@ -218,6 +210,38 @@ func validateSubjectToken(value string) error {
 	return nil
 }
 
+// A publish answered without an error but without a usable export. The
+// provider may have started the export anyway, so a caller must not assume
+// nothing is left to tear down.
+var (
+	ErrNoPublishedVolume = errors.New("returned no published volume")
+	ErrInvalidExport     = errors.New("returned an invalid export")
+)
+
+// ErrUnreadableReply is a reply that arrived but could not be decoded, or
+// carried a schema version this build does not serve. The provider did
+// receive the request, so whatever it asked for may have happened.
+var ErrUnreadableReply = errors.New("provider reply could not be read")
+
+// AccountIDHeader carries the account a contract request acts for. It is the
+// header the control plane's other NATS hops use, so one name attributes a
+// request from the gateway through to the provider.
+const AccountIDHeader = "X-Account-ID"
+
+type accountIDKey struct{}
+
+// WithAccountID returns ctx carrying the account the contract requests made
+// under it act for. An empty accountID leaves them unattributed.
+func WithAccountID(ctx context.Context, accountID string) context.Context {
+	return context.WithValue(ctx, accountIDKey{}, accountID)
+}
+
+// AccountIDFromContext returns the account WithAccountID put on ctx, or "".
+func AccountIDFromContext(ctx context.Context) string {
+	accountID, _ := ctx.Value(accountIDKey{}).(string)
+	return accountID
+}
+
 // NATSProvider drives a provider daemon through the versioned ebs.* contract.
 type NATSProvider struct {
 	conn           *nats.Conn
@@ -239,26 +263,11 @@ func NewNATSProvider(conn *nats.Conn, requestTimeout time.Duration) *NATSProvide
 }
 
 func (p *NATSProvider) GetCapabilities(ctx context.Context, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
-	return p.getCapabilities(ctx, CapabilitiesSubject, req)
-}
-
-// GetNodeCapabilities asks nodeID's own provider what it supports, over
-// NodeCapabilitiesSubject. A provider that predates that subject gets no
-// responder, which a caller must count as no evidence.
-func (p *NATSProvider) GetNodeCapabilities(ctx context.Context, nodeID string, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
-	subject, err := NodeCapabilitiesSubject(nodeID)
-	if err != nil {
-		return nil, err
-	}
-	return p.getCapabilities(ctx, subject, req)
-}
-
-func (p *NATSProvider) getCapabilities(ctx context.Context, subject string, req GetCapabilitiesRequest) (*GetCapabilitiesResponse, error) {
 	if err := checkVersion(req.SchemaVersion); err != nil {
 		return nil, err
 	}
 	var response GetCapabilitiesResponse
-	if err := p.request(ctx, subject, req, &response); err != nil {
+	if err := p.request(ctx, CapabilitiesSubject, req, &response); err != nil {
 		return nil, err
 	}
 	if err := responseError(response.SchemaVersion, response.Error); err != nil {
@@ -373,7 +382,7 @@ func (p *NATSProvider) CreateSnapshot(ctx context.Context, req CreateSnapshotReq
 	if err != nil {
 		return nil, err
 	}
-	if p.conn == nil || !p.conn.IsConnected() {
+	if !p.connected() {
 		return nil, nats.ErrConnectionClosed
 	}
 
@@ -413,7 +422,7 @@ func (p *NATSProvider) CreateSnapshot(ctx context.Context, req CreateSnapshotReq
 	RecordResponseError(waitSpan, msg.Data)
 	var completed CreateSnapshotResponse
 	if err := json.Unmarshal(msg.Data, &completed); err != nil {
-		return nil, fmt.Errorf("decode snapshot completion: %w", err)
+		return nil, fmt.Errorf("decode snapshot completion: %w: %w", ErrUnreadableReply, err)
 	}
 	if err := responseError(completed.SchemaVersion, completed.Error); err != nil {
 		return nil, err
@@ -496,11 +505,11 @@ func (p *NATSProvider) PublishVolume(ctx context.Context, req PublishVolumeReque
 			return nil, err
 		}
 		if response.Published == nil {
-			return nil, fmt.Errorf("%s returned no published volume", subject)
+			return nil, fmt.Errorf("%s %w", subject, ErrNoPublishedVolume)
 		}
 		published := PublishedVolumeFromV1(response.Published)
 		if err := published.Export.Validate(); err != nil {
-			return nil, fmt.Errorf("%s returned an invalid export: %w", subject, err)
+			return nil, fmt.Errorf("%s %w: %w", subject, ErrInvalidExport, err)
 		}
 		return published, nil
 	}
@@ -512,10 +521,10 @@ func (p *NATSProvider) PublishVolume(ctx context.Context, req PublishVolumeReque
 		return nil, err
 	}
 	if response.Published == nil {
-		return nil, fmt.Errorf("%s returned no published volume", subject)
+		return nil, fmt.Errorf("%s %w", subject, ErrNoPublishedVolume)
 	}
 	if err := response.Published.Export.Validate(); err != nil {
-		return nil, fmt.Errorf("%s returned an invalid export: %w", subject, err)
+		return nil, fmt.Errorf("%s %w: %w", subject, ErrInvalidExport, err)
 	}
 	return response.Published, nil
 }
@@ -557,31 +566,64 @@ func (p *NATSProvider) HoldsUnsealedWrites(ctx context.Context, nodeID string, v
 	return response.Held, nil
 }
 
+// connected reports whether a request can be sent now. A reconnecting
+// connection would buffer it and send it on reconnect, after its caller may
+// have given up, so the provider would act with nobody waiting on the result.
+func connected(conn *nats.Conn) bool {
+	return conn != nil && conn.IsConnected()
+}
+
+func (p *NATSProvider) connected() bool { return connected(p.conn) }
+
 func (p *NATSProvider) request(ctx context.Context, subject string, input, output any) error {
-	if p.conn == nil || !p.conn.IsConnected() {
-		return nats.ErrConnectionClosed
+	reply, err := requestReply(ctx, p.conn, subject, p.timeoutFor(subject), input)
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(reply, output); err != nil {
+		return fmt.Errorf("decode %s response: %w: %w", subject, ErrUnreadableReply, err)
+	}
+	return nil
+}
+
+// requestReply sends input on subject as ctx's account, waits up to timeout
+// and returns the raw reply. Its client span records a transport error, an
+// unreadable reply, or the error the reply carries; decoding is the caller's.
+func requestReply(ctx context.Context, conn *nats.Conn, subject string, timeout time.Duration, input any) ([]byte, error) {
+	if !connected(conn) {
+		return nil, nats.ErrConnectionClosed
 	}
 	payload, err := json.Marshal(input)
 	if err != nil {
-		return fmt.Errorf("encode %s request: %w", subject, err)
+		return nil, fmt.Errorf("encode %s request: %w", subject, err)
 	}
 
 	request := &nats.Msg{Subject: subject, Data: payload, Header: nats.Header{}}
-	ctx, span := startClientSpan(ctx, subject, request.Header)
+	accountID := AccountIDFromContext(ctx)
+	if accountID != "" {
+		request.Header.Set(AccountIDHeader, accountID)
+	}
+	ctx, span := startClientSpan(ctx, subject, accountID, request.Header)
 	defer span.End()
 
-	requestCtx, cancel := context.WithTimeout(ctx, p.timeoutFor(subject))
+	requestCtx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
-	msg, err := p.conn.RequestMsgWithContext(requestCtx, request)
+	msg, err := conn.RequestMsgWithContext(requestCtx, request)
 	if err != nil {
 		RecordSpanError(span, err)
-		return fmt.Errorf("request %s: %w", subject, err)
-	}
-	if err := json.Unmarshal(msg.Data, output); err != nil {
-		RecordSpanError(span, err)
-		return fmt.Errorf("decode %s response: %w", subject, err)
+		return nil, fmt.Errorf("request %s: %w", subject, err)
 	}
 	RecordResponseError(span, msg.Data)
+	return msg.Data, nil
+}
+
+// ValidateNodeID reports whether nodeID can address this node's contract
+// subjects. A node that fails it has no fence-notice subject, so a revoked
+// export there would leave its guest running.
+func ValidateNodeID(nodeID string) error {
+	if err := validateSubjectToken(nodeID); err != nil {
+		return fmt.Errorf("node ID cannot address EBS provider subjects: %w", err)
+	}
 	return nil
 }
 
@@ -598,7 +640,7 @@ func (p *NATSProvider) requestOwnerFirst(ctx context.Context, ownerSubject, queu
 
 func responseError(version uint16, providerErr *ProviderError) error {
 	if err := checkVersion(version); err != nil {
-		return err
+		return fmt.Errorf("%w: %w", ErrUnreadableReply, err)
 	}
 	if providerErr != nil {
 		return providerErr

spinifex/providers/ebs/nats_reconnect_test.go

@@ -0,0 +1,58 @@
+package ebsprovider
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A request on a connection that is down fails at once rather than buffering:
+// one flushed on reconnect would run after its caller gave up on it.
+func TestNATSProviderRequestRefusesWhileDisconnected(t *testing.T) {
+	connections := map[string]func(t *testing.T) *nats.Conn{
+		"Reconnecting": func(t *testing.T) *nats.Conn {
+			ns := startNATS(t)
+			nc, err := nats.Connect(ns.ClientURL(), nats.MaxReconnects(-1), nats.ReconnectWait(time.Hour))
+			require.NoError(t, err)
+			t.Cleanup(nc.Close)
+			ns.Shutdown()
+			require.Eventually(t, func() bool { return !nc.IsConnected() }, 5*time.Second, 5*time.Millisecond)
+			require.False(t, nc.IsClosed(), "the connection must still be trying to reconnect")
+			return nc
+		},
+		"Closed": func(t *testing.T) *nats.Conn {
+			nc, err := nats.Connect(startNATS(t).ClientURL())
+			require.NoError(t, err)
+			nc.Close()
+			return nc
+		},
+	}
+	for name, connect := range connections {
+		t.Run(name, func(t *testing.T) {
+			provider := NewNATSProvider(connect(t), 5*time.Second)
+			ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
+			defer cancel()
+
+			_, err := provider.UnpublishVolume(ctx, UnpublishVolumeRequest{
+				Versioned: NewLatestVersioned(), VolumeID: "vol-1", NodeID: "node-down",
+			})
+			require.ErrorIs(t, err, nats.ErrConnectionClosed)
+			assert.NotErrorIs(t, err, context.DeadlineExceeded, "the request must be refused, not left waiting")
+		})
+	}
+}
+
+func startNATS(t *testing.T) *server.Server {
+	t.Helper()
+	ns, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, NoLog: true, NoSigs: true})
+	require.NoError(t, err)
+	go ns.Start()
+	require.True(t, ns.ReadyForConnections(5*time.Second))
+	t.Cleanup(ns.Shutdown)
+	return ns
+}

spinifex/providers/ebs/nats_test.go

@@ -475,6 +475,7 @@ func TestNATSProviderRejectsEmptySuccessResponse(t *testing.T) {
 		respondWith(t, conn, subject, PublishVolumeResponse{Versioned: empty})
 		_, err = provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewLatestVersioned(), VolumeID: "vol-empty", NodeID: "node-empty"})
 		require.EqualError(t, err, subject+" returned no published volume")
+		require.ErrorIs(t, err, ErrNoPublishedVolume)
 	})
 
 	t.Run("CreateSnapshot", func(t *testing.T) {
@@ -512,6 +513,7 @@ func TestNATSProviderRejectsInvalidExport(t *testing.T) {
 			})
 			published, err := provider.PublishVolume(t.Context(), PublishVolumeRequest{Versioned: NewLatestVersioned(), VolumeID: "vol-1", NodeID: tt.node})
 			require.EqualError(t, err, subject+" returned an invalid export: "+tt.wantErr)
+			require.ErrorIs(t, err, ErrInvalidExport)
 			assert.Nil(t, published)
 		})
 	}
@@ -718,8 +720,6 @@ func TestTimeoutForSizesEachSubjectToItsWork(t *testing.T) {
 	require.NoError(t, err)
 	unsealed, err := UnsealedWritesSubject("node-1")
 	require.NoError(t, err)
-	nodeCapabilities, err := NodeCapabilitiesSubject("node-1")
-	require.NoError(t, err)
 
 	// A delete costs one round trip per object, so its duration is a function
 	// of the volume. On prod a 100 GiB volume's delete took 52 s against a 30 s
@@ -740,7 +740,7 @@ func TestTimeoutForSizesEachSubjectToItsWork(t *testing.T) {
 	fast := []string{
 		CapabilitiesSubject, GetVolumeSubject, ListVolumesSubject,
 		ListSnapshotsSubject, ownerDescribe,
-		publish, unsealed, nodeCapabilities,
+		publish, unsealed,
 	}
 	for _, subject := range fast {
 		assert.Equal(t, callerTimeout, p.timeoutFor(subject),

spinifex/providers/ebs/natsserve/handlers_test.go

@@ -126,8 +126,6 @@ func wireVerbs(t *testing.T) []wireVerb {
 	require.NoError(t, err)
 	unsealedSubject, err := ebsprovider.UnsealedWritesSubject("node-1")
 	require.NoError(t, err)
-	nodeCapabilitiesSubject, err := ebsprovider.NodeCapabilitiesSubject("node-1")
-	require.NoError(t, err)
 
 	return []wireVerb{
 		{name: "capabilities", subject: ebsprovider.CapabilitiesSubject,
@@ -156,8 +154,6 @@ func wireVerbs(t *testing.T) []wireVerb {
 			request: ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1", NodeID: "node-1"}},
 		{name: "unsealed", subject: unsealedSubject,
 			request: ebsprovider.HoldsUnsealedWritesRequest{Versioned: ebsprovider.NewVersioned(), NodeID: "node-1", VolumeIDs: []string{"vol-1"}}},
-		{name: "node.capabilities", subject: nodeCapabilitiesSubject,
-			request: ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}},
 	}
 }
 
@@ -737,22 +733,6 @@ func TestServeNodeIDScopesMountSubjects(t *testing.T) {
 	require.ErrorIs(t, err, nats.ErrNoResponders, "another node's mount subject must go unanswered")
 }
 
-// TestGetNodeCapabilitiesIsAnsweredOnlyByThatNode is why the per-node subject
-// exists: the shared one is answered by whichever node's provider is fastest,
-// which says nothing about the provider on the node a caller will mount on.
-func TestGetNodeCapabilitiesIsAnsweredOnlyByThatNode(t *testing.T) {
-	conn := serveProvider(t, ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}), natsserve.Options{NodeID: "node-1"})
-	client := ebsprovider.NewNATSProvider(conn, requestTimeout)
-	request := ebsprovider.GetCapabilitiesRequest{Versioned: ebsprovider.NewVersioned()}
-
-	resp, err := client.GetNodeCapabilities(t.Context(), "node-1", request)
-	require.NoError(t, err)
-	assert.Equal(t, ebsprovider.SupportedSchemaVersions(), resp.ContractVersions)
-
-	_, err = client.GetNodeCapabilities(t.Context(), "node-2", request)
-	require.ErrorIs(t, err, nats.ErrNoResponders, "another node's capabilities subject must go unanswered")
-}
-
 // TestServeRejectsUnroutableNodeID covers the subject-build failure path: an
 // unsubscribable node ID must fail Serve outright rather than leave a server
 // running with the shared subjects subscribed and no mount subject at all.

spinifex/providers/ebs/natsserve/server.go

@@ -36,9 +36,9 @@ type Options struct {
 	// across workers does not apply.
 	NoQueueGroup bool
 
-	// NodeID scopes the per-node subjects (publish, unpublish, unsealed writes
-	// and per-node capabilities) to this node, as a production daemon serves
-	// them. Left empty, Serve subscribes their wildcards and answers for any node.
+	// NodeID scopes the per-node subjects (publish, unpublish and unsealed
+	// writes) to this node, as a production daemon serves them. Left empty,
+	// Serve subscribes their wildcards and answers for any node.
 	NodeID string
 }
 
@@ -135,7 +135,6 @@ func Serve(ctx context.Context, nc *nats.Conn, provider ebsprovider.EBSProvider,
 		{ebsprovider.PublishSubjectWildcard, ebsprovider.PublishSubject, handlePublishVolume(provider)},
 		{ebsprovider.UnpublishSubjectWildcard, ebsprovider.UnpublishSubject, handleUnpublishVolume(provider)},
 		{ebsprovider.UnsealedWritesSubjectWildcard, ebsprovider.UnsealedWritesSubject, handleHoldsUnsealedWrites(provider)},
-		{ebsprovider.NodeCapabilitiesSubjectWildcard, ebsprovider.NodeCapabilitiesSubject, handleCapabilities(provider)},
 	}
 	for _, step := range nodeSteps {
 		subject := step.wildcard

spinifex/providers/ebs/natsserve/server_test.go

@@ -81,3 +81,34 @@ func TestServerSpan_SurvivesAnUntracedCaller(t *testing.T) {
 	assert.Equal(t, "ebs.volume.list", serverSpan.Name())
 	assert.False(t, serverSpan.Parent().IsValid(), "an untraced caller must not leave a dangling parent")
 }
+
+// TestServerSpan_CarriesTheCallersAccount attributes provider work to the
+// account the request acts for, on both halves of the hop, and leaves an
+// unattributed request unattributed.
+func TestServerSpan_CarriesTheCallersAccount(t *testing.T) {
+	recorder := testutil.RecordSpans(t)
+	client := serveMemoryProvider(t)
+
+	ctx := ebsprovider.WithAccountID(t.Context(), "000000000042")
+	_, err := client.ListVolumes(ctx, ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	clientSpan, serverSpan := spansByKind(t, recorder)
+	assert.Equal(t, "000000000042", spanAccount(clientSpan))
+	assert.Equal(t, "000000000042", spanAccount(serverSpan))
+
+	recorder = testutil.RecordSpans(t)
+	_, err = client.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	clientSpan, serverSpan = spansByKind(t, recorder)
+	assert.Empty(t, spanAccount(clientSpan))
+	assert.Empty(t, spanAccount(serverSpan))
+}
+
+func spanAccount(span sdktrace.ReadOnlySpan) string {
+	for _, attr := range span.Attributes() {
+		if attr.Key == "aws.account_id" {
+			return attr.Value.AsString()
+		}
+	}
+	return ""
+}

spinifex/providers/ebs/provider.go

@@ -40,14 +40,13 @@ type Versioned struct {
 	SchemaVersion uint16 `json:"schema_version"`
 }
 
-// NewVersioned returns the oldest representation guaranteed to interoperate
-// during an N/N-1 rollout. Callers may use NewLatestVersioned only after a
-// capability exchange proves the target understands SchemaVersion.
+// NewVersioned returns LegacySchemaVersion, the oldest representation a
+// provider serves, for a request whose shape is the same at every version.
 func NewVersioned() Versioned { return Versioned{SchemaVersion: LegacySchemaVersion} }
 
-// NewLatestVersioned returns the newest representation this build understands.
-// It must not be used merely because this binary is new: a separately released
-// peer may still be serving LegacySchemaVersion.
+// NewLatestVersioned returns SchemaVersion. A cluster runs one contract
+// version, which every provider in it serves, so a request that needs a field
+// only the newest version carries is sent at it without asking first.
 func NewLatestVersioned() Versioned { return Versioned{SchemaVersion: SchemaVersion} }
 
 // SupportsSchemaVersion reports whether this build can safely decode version.

spinifex/providers/ebs/telemetry.go

@@ -3,6 +3,7 @@ package ebsprovider
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"maps"
 	"slices"
 	"strings"
@@ -30,6 +31,9 @@ const (
 	attrNodeID      = "ebs.provider.node_id"
 	attrOwnerRouted = "ebs.provider.owner_routed"
 	attrErrorCode   = "ebs.provider.error_code"
+	// attrAccountID matches the key the gateway and the control plane's own
+	// NATS spans use, so one field follows a request to the provider.
+	attrAccountID = "aws.account_id"
 )
 
 // VerbUnknown is the verb reported for a subject that is not part of the
@@ -115,8 +119,6 @@ func parseSubject(subject string) subjectTarget {
 		return subjectTarget{verb: verbVolumeUnpublish, nodeID: nodeID}
 	case unsealedVerb:
 		return subjectTarget{verb: verbUnsealedWrites, nodeID: nodeID}
-	case nodeCapabilitiesVerb:
-		return subjectTarget{verb: verbCapabilities, nodeID: nodeID}
 	case fenceNoticeVerb:
 		return subjectTarget{verb: verbFenceNotice, nodeID: nodeID}
 	default:
@@ -124,11 +126,16 @@ func parseSubject(subject string) subjectTarget {
 	}
 }
 
-func (t subjectTarget) attributes(subject string) []attribute.KeyValue {
+// attributes describes the target; an empty accountID is left off, as work
+// with no caller must not be credited to anyone.
+func (t subjectTarget) attributes(subject, accountID string) []attribute.KeyValue {
 	attrs := []attribute.KeyValue{
 		attribute.String("messaging.system", "nats"),
 		attribute.String(attrSubject, subject),
 	}
+	if accountID != "" {
+		attrs = append(attrs, attribute.String(attrAccountID, accountID))
+	}
 	if t.volumeID != "" {
 		attrs = append(attrs, attribute.String(attrVolumeID, t.volumeID))
 	}
@@ -156,11 +163,11 @@ func (c headerCarrier) Keys() []string {
 
 // startClientSpan opens a span for an outbound request and injects the trace
 // context into hdr so the serving provider joins the same trace.
-func startClientSpan(ctx context.Context, subject string, hdr nats.Header) (context.Context, trace.Span) {
+func startClientSpan(ctx context.Context, subject, accountID string, hdr nats.Header) (context.Context, trace.Span) {
 	target := parseSubject(subject)
 	ctx, span := otel.Tracer(tracerName).Start(ctx, spanNamePrefix+target.verb,
 		trace.WithSpanKind(trace.SpanKindClient),
-		trace.WithAttributes(target.attributes(subject)...))
+		trace.WithAttributes(target.attributes(subject, accountID)...))
 	otel.GetTextMapPropagator().Inject(ctx, headerCarrier(hdr))
 	return ctx, span
 }
@@ -178,16 +185,18 @@ func startCompletionSpan(ctx context.Context, verb, completionSubject string) (c
 }
 
 // StartServerSpan joins the trace carried in msg's headers and opens a span
-// for the verb its subject names. Callers must End the returned span and
-// should pass the returned context into the work the handler does.
+// for the verb its subject names, attributed to the account the request
+// carries. Callers must End the span and pass the context into the handler.
 func StartServerSpan(ctx context.Context, msg *nats.Msg) (context.Context, trace.Span) {
 	target := parseSubject(msg.Subject)
+	var accountID string
 	if msg.Header != nil {
 		ctx = otel.GetTextMapPropagator().Extract(ctx, headerCarrier(msg.Header))
+		accountID = msg.Header.Get(AccountIDHeader)
 	}
 	return otel.Tracer(tracerName).Start(ctx, spanNamePrefix+target.verb,
 		trace.WithSpanKind(trace.SpanKindServer),
-		trace.WithAttributes(target.attributes(msg.Subject)...))
+		trace.WithAttributes(target.attributes(msg.Subject, accountID)...))
 }
 
 // RecordSpanError marks span failed for a transport or handler error.
@@ -210,9 +219,9 @@ func RecordProviderError(span trace.Span, providerErr *ProviderError) {
 	span.SetStatus(codes.Error, providerErr.Message)
 }
 
-// RecordResponseError tags span with the error an encoded response carries.
-// The response types share no accessor for it, so this re-reads the one field
-// rather than making every verb implement one; it runs only when recording.
+// RecordResponseError tags span with the error an encoded response carries, or
+// fails it for a response that is not readable. The response types share no
+// accessor, so this re-reads the one field; it runs only when recording.
 func RecordResponseError(span trace.Span, data []byte) {
 	if !span.IsRecording() {
 		return
@@ -221,6 +230,7 @@ func RecordResponseError(span trace.Span, data []byte) {
 		Error *ProviderError `json:"error"`
 	}
 	if err := json.Unmarshal(data, &envelope); err != nil {
+		RecordSpanError(span, fmt.Errorf("%w: %w", ErrUnreadableReply, err))
 		return
 	}
 	RecordProviderError(span, envelope.Error)

spinifex/providers/ebs/telemetry_test.go

@@ -32,8 +32,6 @@ func TestSubjectVerb_StripsIDsFromEveryDynamicSubject(t *testing.T) {
 	require.NoError(t, err)
 	unsealed, err := UnsealedWritesSubject(nodeID)
 	require.NoError(t, err)
-	nodeCapabilities, err := NodeCapabilitiesSubject(nodeID)
-	require.NoError(t, err)
 	fenced, err := FenceNoticeSubject(nodeID)
 	require.NoError(t, err)
 
@@ -54,7 +52,6 @@ func TestSubjectVerb_StripsIDsFromEveryDynamicSubject(t *testing.T) {
 		publish:               verbVolumePublish,
 		unpublish:             verbVolumeUnpublish,
 		unsealed:              verbUnsealedWrites,
-		nodeCapabilities:      verbCapabilities,
 		fenced:                verbFenceNotice,
 	}
 

spinifex/providers/ebs/viperblock/conformance_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs/conformance"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	testpredastore "github.com/mulgadc/spinifex/tests/fixtures/predastore"
+	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/require"
 )
@@ -53,24 +54,49 @@ func requireExportTools(t *testing.T) string {
 	return pluginPath
 }
 
+// queueOnlyNode stands a node up whose provider subjects are served on a NATS
+// of its own, with the cluster's lease, dirty and liveness state, so its
+// client reaches it on the queue subjects alone, as when an owner is silent.
+func queueOnlyNode(t *testing.T, node *Config, clusterNC *nats.Conn) ebsprovider.EBSProvider {
+	t.Helper()
+	dirty, err := newVolumeDirty(t.Context(), clusterNC, node.leaseOwner())
+	require.NoError(t, err)
+	node.dirty = dirty
+	node.liveness = newHeartbeatLiveness(clusterNC)
+
+	_, privateURL := setupEmbeddedNATS(t)
+	return ebsprovider.NewNATSProvider(startProviderSubjects(t, node, privateURL), 60*time.Second)
+}
+
+// providerHarness is what inProcessProvider stands up: a client for the
+// cluster, both node names, and the hooks the conformance suite takes.
+type providerHarness struct {
+	newProvider      func(t *testing.T) ebsprovider.EBSProvider
+	nodeID           string
+	otherNodeID      string
+	killExport       func(*testing.T, *ebsprovider.PublishedVolume)
+	revokeExport     func(*testing.T, *ebsprovider.PublishedVolume) ebsprovider.FenceNotice
+	queueOnOtherNode func(t *testing.T) ebsprovider.EBSProvider
+}
+
 // inProcessProvider stands two nodes of the provider handlers up over one
-// embedded NATS and one predastore, and returns a client for them plus both
-// node names. Everything is torn down with the test.
+// embedded NATS and one predastore. Everything is torn down with the test.
 //
 // Two nodes rather than one because the exclusion this provider advertises is
 // cluster-scoped, and a single node cannot exercise it: the second publish
 // would go to a subject nobody serves and time out instead of being refused.
 // Both share the NATS the volume leases live in and the bucket the volumes
 // live in, so a second opener is refused by the lease, not by the harness.
-// The last return kills a published export's nbdkit behind the handlers' back.
-func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider, string, string, func(*testing.T, *ebsprovider.PublishedVolume)) {
+// The hooks kill a published export's nbdkit behind the handlers' back, and
+// revoke one as a lost lease does.
+func inProcessProvider(t *testing.T) providerHarness {
 	t.Helper()
 	pluginPath := requireExportTools(t)
 
 	fixture := testpredastore.Start(t)
 	_, natsURL := setupEmbeddedNATS(t)
 
-	newNode := func(name string) *Config {
+	newNode := func(t *testing.T, name string) *Config {
 		cfg := setupTestConfig(t, natsURL)
 		cfg.S3Host = "https://" + fixture.Host
 		cfg.Bucket = testpredastore.DefaultBucket
@@ -88,8 +114,8 @@ func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider
 		return cfg
 	}
 
-	first := newNode("node-1")
-	second := newNode("node-2")
+	first := newNode(t, "node-1")
+	second := newNode(t, "node-2")
 	nc := startProviderSubjects(t, first, natsURL)
 	startProviderSubjects(t, second, natsURL)
 
@@ -100,21 +126,42 @@ func inProcessProvider(t *testing.T) (func(t *testing.T) ebsprovider.EBSProvider
 	_, err = js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
 	require.NoError(t, err)
 
-	killExport := func(t *testing.T, published *ebsprovider.PublishedVolume) {
+	holder := func(t *testing.T, published *ebsprovider.PublishedVolume) (*Config, MountedVolume) {
 		t.Helper()
 		for _, node := range []*Config{first, second} {
 			if mv, ok := findMountedVolume(node, published.VolumeID); ok && node.NodeName == published.NodeID {
-				require.NoError(t, hostprocess.ForceKillProcess(mv.PID, 5*time.Second))
-				return
+				return node, mv
 			}
 		}
 		t.Fatalf("no mounted entry for %s on %s", published.VolumeID, published.NodeID)
+		return nil, MountedVolume{}
 	}
 
-	return func(t *testing.T) ebsprovider.EBSProvider {
-		t.Helper()
-		return ebsprovider.NewNATSProvider(nc, 60*time.Second)
-	}, first.NodeName, second.NodeName, killExport
+	return providerHarness{
+		newProvider: func(t *testing.T) ebsprovider.EBSProvider {
+			t.Helper()
+			return ebsprovider.NewNATSProvider(nc, 60*time.Second)
+		},
+		nodeID:      first.NodeName,
+		otherNodeID: second.NodeName,
+		killExport: func(t *testing.T, published *ebsprovider.PublishedVolume) {
+			t.Helper()
+			_, mv := holder(t, published)
+			require.NoError(t, hostprocess.ForceKillProcess(mv.PID, 5*time.Second))
+		},
+		revokeExport: func(t *testing.T, published *ebsprovider.PublishedVolume) ebsprovider.FenceNotice {
+			t.Helper()
+			node, _ := holder(t, published)
+			return conformance.CaptureFenceNotice(t, nc, published.NodeID, func() error {
+				node.onVolumeLeaseLost(t.Context(), published.VolumeID, leaseLostStalled)
+				return nil
+			})
+		},
+		queueOnOtherNode: func(t *testing.T) ebsprovider.EBSProvider {
+			t.Helper()
+			return queueOnlyNode(t, newNode(t, "node-3"), nc)
+		},
+	}
 }
 
 // runPrefix names one run's volumes so it cannot meet an earlier run's
@@ -126,12 +173,14 @@ func runPrefix() string {
 // TestViperblockdConformance judges the provider handlers by the same suite
 // MemoryProvider and qemunbdd answer to.
 func TestViperblockdConformance(t *testing.T) {
-	newProvider, nodeID, otherNodeID, killExport := inProcessProvider(t)
-	conformance.RunSuiteWithConfig(t, newProvider, conformance.SuiteConfig{
-		NamePrefix:  runPrefix(),
-		NodeID:      nodeID,
-		OtherNodeID: otherNodeID,
-		KillExport:  killExport,
+	h := inProcessProvider(t)
+	conformance.RunSuiteWithConfig(t, h.newProvider, conformance.SuiteConfig{
+		NamePrefix:       runPrefix(),
+		NodeID:           h.nodeID,
+		OtherNodeID:      h.otherNodeID,
+		KillExport:       h.killExport,
+		RevokeExport:     h.revokeExport,
+		QueueOnOtherNode: h.queueOnOtherNode,
 	})
 }
 
@@ -139,9 +188,9 @@ func TestViperblockdConformance(t *testing.T) {
 // cannot: whether the nbdkit export viperblockd publishes is usable by libnbd,
 // which knows nothing about this codebase.
 func TestViperblockdNBDClient(t *testing.T) {
-	newProvider, nodeID, _, _ := inProcessProvider(t)
-	conformance.RunNBDClientSuiteWithConfig(t, newProvider, conformance.NBDClientConfig{
-		NodeID:       nodeID,
+	h := inProcessProvider(t)
+	conformance.RunNBDClientSuiteWithConfig(t, h.newProvider, conformance.NBDClientConfig{
+		NodeID:       h.nodeID,
 		VolumePrefix: "vol-" + runPrefix() + "nbd",
 	})
 }

spinifex/providers/ebs/viperblock/provider_contract_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/internal/testkit/deadpid"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/nbd"
@@ -176,23 +177,6 @@ func TestProviderHandlers_PublishVolume_RepublishRepliesInRequestVersion(t *test
 	}
 }
 
-// TestProviderHandlers_NodeCapabilitiesListsServedVersions pins the per-node
-// subject to the same contract versions this build serves.
-func TestProviderHandlers_NodeCapabilitiesListsServedVersions(t *testing.T) {
-	_, natsURL := setupEmbeddedNATS(t)
-	cfg := setupTestConfig(t, natsURL)
-	nc := startProviderSubjects(t, cfg, natsURL)
-
-	subject, err := ebsprovider.NodeCapabilitiesSubject(cfg.NodeName)
-	require.NoError(t, err)
-	msg := requestProvider(t, nc, subject, marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion}))
-
-	var resp ebsprovider.GetCapabilitiesResponse
-	require.NoError(t, json.Unmarshal(msg.Data, &resp))
-	require.Nil(t, resp.Error)
-	assert.Equal(t, ebsprovider.SupportedSchemaVersions(), resp.ContractVersions)
-}
-
 // TestProviderHandlers_UnpublishNoSealAboveServedVersionRefused covers a caller
 // newer than this build: a no_seal it cannot vouch for is refused, and the
 // export is neither torn down nor sealed.
@@ -252,6 +236,9 @@ func TestPublishError_Codes(t *testing.T) {
 			assert.Equal(t, tc.want, got.Code)
 		})
 	}
+	// Callers still recognise held-elsewhere by its text, so the message keeps it.
+	heldElsewhere := publishError(ebsprovider.NoSealSchemaVersion, "vol-publisherror01", leaseHeld)
+	assert.Contains(t, heldElsewhere.Message, ec2v1.VolumeHeldElsewhereText)
 	assert.True(t, ebsprovider.ErrorCodeUnavailable.Retryable(), "a backing store that is not ready must read as retry-later")
 }
 

spinifex/providers/ebs/viperblock/provider_handlers.go

@@ -53,12 +53,12 @@ var providerObjectStoreFactory = func(cfg *Config) objectstore.ObjectStore {
 // viperblock engine construction out of the EC2 control-plane handlers and
 // into the storage daemon that owns BaseDir and the mounted-volume registry.
 //
-// PublishVolume/UnpublishVolume are node-addressed (ebsprovider.PublishSubject
-// / UnpublishSubject already route to one node), so they are only registered
-// when cfg.NodeName is set; there is no queue-group fallback the way the
-// legacy ebs.mount/ebs.unmount subjects have.
+// It refuses a cfg.NodeName that cannot address the node-addressed subjects.
 // Exported so harnesses can put a real provider behind the control plane.
 func RegisterProviderSubjects(cfg *Config, nc *nats.Conn) error {
+	if err := cfg.validateNodeName(); err != nil {
+		return err
+	}
 	// The lease store has to exist before any subject is served: a handler
 	// that reaches an engine open without one refuses, and refusing every
 	// publish is a worse failure than not starting.
@@ -114,11 +114,6 @@ func RegisterProviderSubjects(cfg *Config, nc *nats.Conn) error {
 		return fmt.Errorf("subscribe to %s: %w", snapshotCreateWildcard, err)
 	}
 
-	if cfg.NodeName == "" {
-		slog.Warn("ebs.provider: NodeName is empty, this node cannot serve PublishVolume/UnpublishVolume attachments")
-		return nil
-	}
-
 	publishSubject, err := ebsprovider.PublishSubject(cfg.NodeName)
 	if err != nil {
 		return fmt.Errorf("build publish subject: %w", err)
@@ -149,16 +144,6 @@ func RegisterProviderSubjects(cfg *Config, nc *nats.Conn) error {
 		return fmt.Errorf("subscribe to %s: %w", unsealedSubject, err)
 	}
 
-	// The queue subject is answered by any node's provider; this one only by
-	// this node's, so a daemon can learn what its own provider serves.
-	nodeCapabilitiesSubject, err := ebsprovider.NodeCapabilitiesSubject(cfg.NodeName)
-	if err != nil {
-		return fmt.Errorf("build node capabilities subject: %w", err)
-	}
-	if _, err := nc.Subscribe(nodeCapabilitiesSubject, tracedProviderHandler(handleProviderCapabilities)); err != nil {
-		return fmt.Errorf("subscribe to %s: %w", nodeCapabilitiesSubject, err)
-	}
-
 	return nil
 }
 
@@ -1141,13 +1126,9 @@ func handleCreateSnapshot(ctx context.Context, cfg *Config, nc *nats.Conn, msg *
 		},
 	})
 
-	// The caller is responsible for draining the volume (flushing guest
-	// writes to the live checkpoint) before requesting a snapshot; that
-	// coordination goes through the guest over ec2.cmd.<instanceID> and
-	// stays a control-plane concern this provider boundary does not reach.
-	// WithoutCancel keeps the trace context while dropping the deadline: the
-	// work has to outlive the request it was accepted by, and it still has to
-	// be attributable to it.
+	// The drain runs in the background with the snapshot, so only the accept
+	// reply is freed of it; a caller waiting on completion still waits for it.
+	// WithoutCancel keeps the trace context but drops the request's deadline.
 	go completeCreateSnapshot(context.WithoutCancel(ctx), cfg, nc, req, operationID, completionSubject)
 }
 
@@ -1188,7 +1169,7 @@ func snapshotErrorCode(err error) ebsprovider.ErrorCode {
 		return ebsprovider.ErrorCodeNotFound
 	case errors.Is(err, errSnapshotExistsElsewhere):
 		return ebsprovider.ErrorCodeAlreadyExists
-	case errors.Is(err, errVolumeLeaseHeld), errors.Is(err, errNoVolumeLeaseStore):
+	case errors.Is(err, errVolumeLeaseHeld), errors.Is(err, errNoVolumeLeaseStore), errors.Is(err, errExportSettling):
 		// Exclusive access could not be established. The holder gives the
 		// volume up on detach, so this is worth repeating rather than the
 		// permanent failure an internal error reads as.
@@ -1216,26 +1197,36 @@ func checkSnapshotIDFree(vb *viperblock.VB, volumeID, snapshotID string) error {
 }
 
 // snapshotVolumeEngine creates snapshotID off volumeID's live checkpoint,
-// preferring an already-mounted VB over opening a second engine on the same
-// volume. Draining the volume so the checkpoint is current is the caller's
-// responsibility (see handleCreateSnapshot).
-//
-// The unmounted branch opens through openVolumeVB, so it holds the volume's
-// cluster-wide lease for as long as the engine is open: a snapshot taken
-// beside a live writer on another node is two engines on one volume.
+// preferring an already-mounted VB, drained first so the checkpoint is
+// current, over opening a second engine on the same volume.
 func snapshotVolumeEngine(ctx context.Context, cfg *Config, volumeID, snapshotID string) (*ebsprovider.Snapshot, error) {
 	if mv, ok := findMountedVolume(cfg, volumeID); ok && mv.VB != nil {
+		if err := drainMountedExport(ctx, cfg, volumeID); err != nil {
+			return nil, err
+		}
 		return createSnapshotWithVB(mv.VB, volumeID, snapshotID)
 	}
 
-	vb, lease, err := openVolumeVB(ctx, cfg, volumeID)
+	// The lease refuses a volume another node holds. It is reentrant here, so
+	// an export this node runs but has not registered is excluded or drained
+	// separately, and the engine opens only once that is settled.
+	lease, err := cfg.acquireVolumeLease(ctx, volumeID)
 	if err != nil {
 		return nil, err
 	}
-	defer func() {
-		vb.Detach()
-		cfg.releaseVolumeLease(ctx, lease)
-	}()
+	defer cfg.releaseVolumeLease(ctx, lease)
+
+	lock, err := excludeLocalWriter(ctx, cfg, volumeID)
+	if err != nil {
+		return nil, err
+	}
+	defer releaseWriterLock(ctx, lock, volumeID)
+
+	vb, err := openLeasedVolumeVB(ctx, cfg, volumeID, lease)
+	if err != nil {
+		return nil, err
+	}
+	defer vb.Detach()
 	return createSnapshotWithVB(vb, volumeID, snapshotID)
 }
 
@@ -1813,8 +1804,8 @@ func unmountVolume(ctx context.Context, cfg *Config, volumeName string) (viperbl
 
 // handlePublishVolume serves ebs.provider.v1.<node>.mount, the provider-neutral
 // front for the same nbdkit mount path ebs.mount uses. It is node-addressed
-// (RegisterProviderSubjects only subscribes it when cfg.NodeName is set), so
-// unlike the legacy handler there is no queue-group fallback to reason about.
+// (subscribed on cfg.NodeName's subject), so unlike the legacy handler there
+// is no queue-group fallback to reason about.
 func handlePublishVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *nats.Msg) {
 	var req ebsprovider.PublishVolumeRequest
 	err := json.Unmarshal(msg.Data, &req)
@@ -1929,7 +1920,7 @@ func tearDownPublication(ctx context.Context, cfg *Config, req ebsprovider.Unpub
 	if err != nil {
 		// Nothing mounted here is already the state unpublish asks for, so a
 		// retry after a request that completed server-side converges rather
-		// than erroring, matching the legacy path's unmountResponseError.
+		// than erroring.
 		if unmountResp.NotFound {
 			return unmountResp.Reaped, nil
 		}

spinifex/providers/ebs/viperblock/provider_handlers_test.go

@@ -154,7 +154,6 @@ func providerSubjectCases() []providerSubjectCase {
 			"node_id":    "test-node",
 			"volume_ids": []string{"vol-testunsealed01"},
 		}},
-		{"node_capabilities", "ebs.provider.v1.test-node.capabilities", map[string]any{}},
 	}
 }
 
@@ -1007,9 +1006,8 @@ func TestProviderHandlers_UnpublishVolume_RemovesMountedVolume(t *testing.T) {
 }
 
 // TestProviderHandlers_UnpublishVolume_AbsentIsIdempotent covers a volume this
-// node never mounted. The legacy path treats that as a successful seal (see
-// unmountResponseError), and so must this one: a retry after a request that
-// timed out client-side but completed server-side has to converge.
+// node never mounted, which counts as a successful seal: a retry after a
+// request that timed out client-side but completed server-side must converge.
 func TestProviderHandlers_UnpublishVolume_AbsentIsIdempotent(t *testing.T) {
 	_, natsURL := setupEmbeddedNATS(t)
 	cfg := setupTestConfig(t, natsURL)
@@ -1029,29 +1027,29 @@ func TestProviderHandlers_UnpublishVolume_AbsentIsIdempotent(t *testing.T) {
 	assert.Nil(t, resp.Error, "unpublishing a volume this node never mounted must succeed")
 }
 
-// TestProviderHandlers_PublishUnpublish_RequireNodeName covers
-// RegisterProviderSubjects' NodeName gate: PublishVolume/UnpublishVolume must
-// not be registered when cfg.NodeName is empty (PublishSubject has no node to
-// address), while every other ebs.provider.v1.* subject still is.
-func TestProviderHandlers_PublishUnpublish_RequireNodeName(t *testing.T) {
+// TestProviderHandlers_RefuseAnUnaddressableNodeName keeps viperblockd from
+// serving where the node-addressed subjects, the fence notice among them, have
+// no subject: a fence there would leave its guest running.
+func TestProviderHandlers_RefuseAnUnaddressableNodeName(t *testing.T) {
 	_, natsURL := setupEmbeddedNATS(t)
-	cfgNoNode := setupTestConfig(t, natsURL)
-	cfgNoNode.NodeName = ""
-	ncNoNode := startProviderSubjects(t, cfgNoNode, natsURL)
+	for _, node := range []string{"", "node.a", "node-*"} {
+		t.Run(node, func(t *testing.T) {
+			nc, err := nats.Connect(natsURL)
+			require.NoError(t, err)
+			t.Cleanup(nc.Close)
 
-	_, natsURL2 := setupEmbeddedNATS(t)
-	cfgNamed := setupTestConfig(t, natsURL2)
-	ncNamed := startProviderSubjects(t, cfgNamed, natsURL2)
+			register := setupTestConfig(t, natsURL)
+			register.NodeName = node
+			err = RegisterProviderSubjects(register, nc)
+			require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
+			assert.Contains(t, err.Error(), "viperblockd node name")
 
-	assert.Equal(t, ncNamed.NumSubscriptions()-4, ncNoNode.NumSubscriptions(),
-		"an empty NodeName must skip exactly the publish, unpublish, unsealed-writes and node capabilities subscriptions")
-
-	// The other provider subjects must still work without a node name.
-	body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion})
-	msg := requestProvider(t, ncNoNode, ebsprovider.CapabilitiesSubject, body)
-	var resp ebsprovider.GetCapabilitiesResponse
-	require.NoError(t, json.Unmarshal(msg.Data, &resp))
-	assert.Nil(t, resp.Error)
+			run := setupTestConfig(t, natsURL)
+			run.NodeName = node
+			require.ErrorIs(t, run.Run(nc), ebsprovider.ErrInvalidArgument)
+			assert.Zero(t, nc.NumSubscriptions(), "a refused start must serve nothing")
+		})
+	}
 }
 
 // TestProviderHandlers_CreateVolume_SeedValidation covers the guards that must

spinifex/providers/ebs/viperblock/snapshot_drain.go

@@ -0,0 +1,129 @@
+package viperblock
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/viperblock/viperblock"
+)
+
+const (
+	// exportDrainDialTimeout bounds the connect to the plugin's drain socket.
+	// The socket is local, so a slow connect is a wedged plugin.
+	exportDrainDialTimeout = time.Second
+
+	// exportDrainAckTimeout bounds the plugin's flush of its WAL and live
+	// checkpoint to the backend, which scales with the dirty set.
+	exportDrainAckTimeout = 30 * time.Second
+
+	// maxUnixSocketPath is sun_path's size. The plugin cannot listen on a
+	// longer path, so a drain socket there never exists.
+	maxUnixSocketPath = 108
+)
+
+// errExportDrainFailed is a drain the plugin was asked for and did not
+// confirm, so the live checkpoint cannot be shown to be current.
+var errExportDrainFailed = errors.New("export drain failed")
+
+// errExportSettling is an export whose client is connecting or disconnecting:
+// it holds the volume lock without serving the drain socket, so its writes are
+// neither drainable nor yet sealed. It clears once the client settles.
+var errExportSettling = errors.New("export is between connections")
+
+// drainMountedExport asks the nbdkit plugin serving volumeID on this node to
+// flush its WAL and live checkpoint to the backend before a snapshot reads
+// them. The plugin serves the socket only while a client is connected.
+func drainMountedExport(ctx context.Context, cfg *Config, volumeID string) error {
+	sockPath := filepath.Join(cfg.BaseDir, volumeID, "snapshot.sock")
+	if len(sockPath) >= maxUnixSocketPath {
+		return requireNoConnectedClient(ctx, cfg, volumeID, "")
+	}
+	start := time.Now()
+	conn, err := net.DialTimeout("unix", sockPath, exportDrainDialTimeout)
+	if errors.Is(err, fs.ErrNotExist) {
+		return requireNoConnectedClient(ctx, cfg, volumeID, sockPath)
+	}
+	if err != nil {
+		return fmt.Errorf("%w: %s: %w", errExportDrainFailed, volumeID, err)
+	}
+	defer conn.Close()
+
+	_ = conn.SetDeadline(time.Now().Add(exportDrainAckTimeout))
+	buf := make([]byte, 8)
+	n, err := conn.Read(buf)
+	if err != nil {
+		return fmt.Errorf("%w: %s: read ack: %w", errExportDrainFailed, volumeID, err)
+	}
+	if !strings.HasPrefix(string(buf[:n]), "OK") {
+		return fmt.Errorf("%w: %s: ack %q", errExportDrainFailed, volumeID, string(buf[:n]))
+	}
+	slog.InfoContext(ctx, "ebs.provider.snapshot.create: drained export", "volume", volumeID, "drain_ms", otelsetup.Millis(time.Since(start)))
+	return nil
+}
+
+// requireNoConnectedClient proves an export serving no drain socket has nothing
+// buffered. sockPath is empty when the socket path is too long for the plugin
+// to listen on, so a held lock there never clears.
+func requireNoConnectedClient(ctx context.Context, cfg *Config, volumeID, sockPath string) error {
+	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
+	if errors.Is(err, viperblock.ErrVolumeLocked) {
+		if sockPath == "" {
+			return fmt.Errorf("%w: %s: client connected with no drain socket", errExportDrainFailed, volumeID)
+		}
+		slog.WarnContext(ctx, "ebs.provider.snapshot.create: volume locked with no drain socket, refusing as retry-later",
+			"volume", volumeID, "socket", sockPath)
+		return fmt.Errorf("%w: %s", errExportSettling, volumeID)
+	}
+	if err != nil {
+		return fmt.Errorf("%w: %s: %w", errExportDrainFailed, volumeID, err)
+	}
+	if err := viperblock.ReleaseVolumeLock(lock); err != nil {
+		return fmt.Errorf("%w: %s: %w", errExportDrainFailed, volumeID, err)
+	}
+	return requireLastClientSealed(cfg, volumeID)
+}
+
+// requireLastClientSealed refuses an export whose last client left WAL behind
+// without a seal receipt newer than it. The plugin releases the lock after a
+// failed seal too, so a free lock alone does not show the writes reached the backend.
+func requireLastClientSealed(cfg *Config, volumeID string) error {
+	if isAuxVolume(volumeID) || !volumeHoldsLocalWAL(cfg, volumeID) || sealReceiptCoversWAL(cfg, volumeID) {
+		return nil
+	}
+	return fmt.Errorf("%w: %s: last client left unsealed WAL", errExportDrainFailed, volumeID)
+}
+
+// excludeLocalWriter proves no export on this node holds writes the backend
+// lacks before a snapshot opens its own engine. The returned lock keeps a new
+// writer out until the snapshot is read; it is nil when a live export was drained.
+func excludeLocalWriter(ctx context.Context, cfg *Config, volumeID string) (*os.File, error) {
+	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
+	if errors.Is(err, viperblock.ErrVolumeLocked) {
+		return nil, drainMountedExport(ctx, cfg, volumeID)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s: %w", errExportDrainFailed, volumeID, err)
+	}
+	if err := requireLastClientSealed(cfg, volumeID); err != nil {
+		releaseWriterLock(ctx, lock, volumeID)
+		return nil, err
+	}
+	return lock, nil
+}
+
+// releaseWriterLock gives up a lock excludeLocalWriter took. A failed release
+// leaves only the kernel to drop it at exit, so it is logged, not returned.
+func releaseWriterLock(ctx context.Context, lock *os.File, volumeID string) {
+	if err := viperblock.ReleaseVolumeLock(lock); err != nil {
+		slog.WarnContext(ctx, "ebs.provider.snapshot.create: failed to release volume lock", "volume", volumeID, "err", err)
+	}
+}

spinifex/providers/ebs/viperblock/snapshot_drain_test.go

@@ -0,0 +1,450 @@
+package viperblock
+
+import (
+	"encoding/json"
+	"io/fs"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/viperblock/viperblock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// mountedSnapshotFixture is a node with volumeID mounted over a real engine,
+// whose base dir is where testutil's drain socket stand-in listens.
+func mountedSnapshotFixture(t *testing.T, volumeID string) (cfg *Config, dataDir string, vb *viperblock.VB) {
+	t.Helper()
+	dataDir = testutil.SocketTempDir(t)
+	cfg, vb = mountedAt(t, volumeID, filepath.Join(dataDir, "viperblock"))
+	return cfg, dataDir, vb
+}
+
+// mountedAt is a node with volumeID mounted over a real engine under baseDir.
+func mountedAt(t *testing.T, volumeID, baseDir string) (*Config, *viperblock.VB) {
+	t.Helper()
+	vb := createTestVBWithState(t, volumeID)
+	t.Cleanup(func() {
+		vb.StopChunkUploader()
+		vb.StopWALSyncer()
+	})
+	cfg := &Config{
+		NodeName:       "node-a",
+		BaseDir:        baseDir,
+		MountedVolumes: []MountedVolume{{Name: volumeID, VB: vb}},
+	}
+	require.NoError(t, os.MkdirAll(cfg.BaseDir, 0o750))
+	return cfg, vb
+}
+
+// leaveUnsealedWAL stands in for a client whose disconnect seal failed: the
+// plugin keeps the volume's local WAL and writes no receipt.
+func leaveUnsealedWAL(t *testing.T, cfg *Config, volumeID string) {
+	t.Helper()
+	require.NoError(t, os.MkdirAll(filepath.Join(cfg.BaseDir, volumeID, "wal", "chunks"), 0o750))
+}
+
+func requireNoSnapshot(t *testing.T, vb *viperblock.VB, snapshotID string) {
+	t.Helper()
+	exists, err := snapshotConfigExists(vb, snapshotID)
+	require.NoError(t, err)
+	assert.False(t, exists, "a snapshot that could not be shown current must not be written")
+}
+
+// The snapshot reads the checkpoint the plugin flushes, so it must not start
+// until the plugin has acknowledged the drain.
+func TestSnapshotVolumeEngine_MountedVolumeIsDrainedBeforeTheSnapshot(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainfirst", "snap-drainfirst00001"
+	cfg, dataDir, vb := mountedSnapshotFixture(t, volumeID)
+	accepted, release := testutil.StartBlockingDrainSocket(t, dataDir, volumeID, "OK\n")
+
+	type result struct {
+		snapshot *ebsprovider.Snapshot
+		err      error
+	}
+	done := make(chan result, 1)
+	go func() {
+		snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+		done <- result{snapshot, err}
+	}()
+
+	select {
+	case <-accepted:
+	case <-time.After(5 * time.Second):
+		t.Fatal("a mounted volume's snapshot never asked the plugin to drain")
+	}
+	requireNoSnapshot(t, vb, snapshotID)
+	release()
+
+	select {
+	case got := <-done:
+		require.NoError(t, got.err)
+		assert.Equal(t, ebsprovider.SnapshotStateCompleted, got.snapshot.State)
+	case <-time.After(10 * time.Second):
+		t.Fatal("snapshot did not complete after the drain acked")
+	}
+}
+
+// The control plane issues no drain of its own, so this one is the only flush
+// an attached volume's snapshot gets, and it must happen exactly once.
+func TestSnapshotVolumeEngine_MountedVolumeIsDrainedOnce(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainonce", "snap-drainonce000001"
+	cfg, dataDir, _ := mountedSnapshotFixture(t, volumeID)
+	drains := testutil.StartDrainSocket(t, dataDir, volumeID, "OK\n")
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snapshot.State)
+	assert.Equal(t, 1, drains())
+}
+
+// A plugin that answers without confirming the flush leaves the checkpoint
+// unproven, so the snapshot fails rather than capturing older data.
+func TestSnapshotVolumeEngine_RefusedDrainFailsTheSnapshot(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainerr", "snap-drainerr0000001"
+	cfg, dataDir, vb := mountedSnapshotFixture(t, volumeID)
+	testutil.StartDrainSocket(t, dataDir, volumeID, "ERR\n")
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.Nil(t, snapshot)
+	assert.Equal(t, ebsprovider.ErrorCodeInternal, snapshotErrorCode(err))
+	requireNoSnapshot(t, vb, snapshotID)
+}
+
+// A socket file nobody listens on is a plugin that died mid-connection, whose
+// writes were never drained.
+func TestSnapshotVolumeEngine_StaleDrainSocketFailsTheSnapshot(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainstale", "snap-drainstale00001"
+	cfg, _, vb := mountedSnapshotFixture(t, volumeID)
+	sockDir := filepath.Join(cfg.BaseDir, volumeID)
+	require.NoError(t, os.MkdirAll(sockDir, 0o750))
+	ln, err := net.Listen("unix", filepath.Join(sockDir, "snapshot.sock"))
+	require.NoError(t, err)
+	ln.(*net.UnixListener).SetUnlinkOnClose(false)
+	require.NoError(t, ln.Close())
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.Nil(t, snapshot)
+	requireNoSnapshot(t, vb, snapshotID)
+}
+
+// The plugin serves the socket only while a client is connected, and drains
+// on disconnect. An export with no client has nothing buffered to flush.
+func TestSnapshotVolumeEngine_ExportWithoutAClientNeedsNoDrain(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainidle", "snap-drainidle000001"
+	cfg, _, _ := mountedSnapshotFixture(t, volumeID)
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snapshot.State)
+}
+
+// A client between connections holds the volume lock without serving the
+// socket: its writes are neither drainable nor sealed yet, which clears soon.
+func TestSnapshotVolumeEngine_ExportBetweenConnectionsIsRetryLater(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainsettle", "snap-drainsettle0001"
+	cfg, _, vb := mountedSnapshotFixture(t, volumeID)
+	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = viperblock.ReleaseVolumeLock(lock) })
+	logs := captureLogs(t)
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportSettling)
+	assert.Nil(t, snapshot)
+	assert.Equal(t, ebsprovider.ErrorCodeUnavailable, snapshotErrorCode(err))
+	requireNoSnapshot(t, vb, snapshotID)
+	// A plugin that never serves its socket refuses every retry, so each
+	// refusal is logged with the socket an operator would look for.
+	sockPath := filepath.Join(cfg.BaseDir, volumeID, "snapshot.sock")
+	assert.True(t, slices.ContainsFunc(strings.Split(logs.String(), "\n"), func(line string) bool {
+		return strings.Contains(line, "level=WARN") && strings.Contains(line, "volume="+volumeID) &&
+			strings.Contains(line, "socket="+sockPath)
+	}), "the refusal must be logged at WARN with the volume and socket: %s", logs.String())
+}
+
+// The plugin releases the lock even when its disconnect seal fails, so a free
+// lock with WAL left behind and no receipt is writes that never reached the
+// backend, and the checkpoint a snapshot would read predates them.
+func TestSnapshotVolumeEngine_UnsealedLastClientFailsTheSnapshot(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainunsealed", "snap-drainunsealed01"
+	cfg, _, vb := mountedSnapshotFixture(t, volumeID)
+	leaveUnsealedWAL(t, cfg, volumeID)
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.Nil(t, snapshot)
+	assert.Equal(t, ebsprovider.ErrorCodeInternal, snapshotErrorCode(err))
+	requireNoSnapshot(t, vb, snapshotID)
+}
+
+// A seal receipt is the plugin's proof the last client's writes reached the
+// backend, so local WAL beside it does not block the snapshot.
+func TestSnapshotVolumeEngine_SealedLastClientProceeds(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainsealed", "snap-drainsealed0001"
+	cfg, _, _ := mountedSnapshotFixture(t, volumeID)
+	leaveUnsealedWAL(t, cfg, volumeID)
+	receipt, err := json.Marshal(sealReceipt{Volume: volumeID, PID: 1, SealedAt: time.Now()})
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(sealReceiptPath(cfg.BaseDir, volumeID), receipt, 0o600))
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snapshot.State)
+	assert.FileExists(t, sealReceiptPath(cfg.BaseDir, volumeID), "the receipt belongs to the unmount it attests to")
+}
+
+// A socket path the plugin could not have listened on never appears, so a
+// connected client there can never be drained: that is permanent, not retry.
+func TestSnapshotVolumeEngine_ConnectedClientWithUnservableSocketFails(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainlongpath", "snap-drainlongpath01"
+	baseDir := filepath.Join(t.TempDir(), strings.Repeat("d", maxUnixSocketPath))
+	cfg, vb := mountedAt(t, volumeID, baseDir)
+	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = viperblock.ReleaseVolumeLock(lock) })
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.NotErrorIs(t, err, errExportSettling)
+	assert.Nil(t, snapshot)
+	assert.Equal(t, ebsprovider.ErrorCodeInternal, snapshotErrorCode(err))
+	requireNoSnapshot(t, vb, snapshotID)
+}
+
+// A plugin that closes the connection without an ack has not confirmed the
+// flush, so the snapshot fails.
+func TestSnapshotVolumeEngine_DrainClosedWithoutAckFailsTheSnapshot(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainnoack", "snap-drainnoack00001"
+	cfg, dataDir, vb := mountedSnapshotFixture(t, volumeID)
+	testutil.StartDrainSocket(t, dataDir, volumeID, "")
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.Nil(t, snapshot)
+	requireNoSnapshot(t, vb, snapshotID)
+}
+
+// lockProbingS3Host is a backend that refuses every request, recording for
+// each whether a new writer could have taken volumeID's lock at that moment.
+type lockProbingS3Host struct {
+	url      string
+	requests atomic.Int32
+	lockable atomic.Int32
+}
+
+func startLockProbingS3Host(t *testing.T, baseDir, volumeID string) *lockProbingS3Host {
+	t.Helper()
+	probe := &lockProbingS3Host{}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		probe.requests.Add(1)
+		if lock, err := viperblock.AcquireVolumeLock(baseDir, volumeID); err == nil {
+			probe.lockable.Add(1)
+			_ = viperblock.ReleaseVolumeLock(lock)
+		}
+		w.WriteHeader(http.StatusBadRequest)
+	}))
+	t.Cleanup(srv.Close)
+	probe.url = srv.URL
+	return probe
+}
+
+// unmountedSnapshotFixture is the volume's own host with nothing registered
+// for it, so a snapshot takes the open-an-engine branch.
+func unmountedSnapshotFixture(t *testing.T, volumeID string) (*Config, string, *lockProbingS3Host) {
+	t.Helper()
+	_, natsURL := setupEmbeddedNATS(t)
+	dataDir := testutil.SocketTempDir(t)
+	baseDir := filepath.Join(dataDir, "viperblock")
+	require.NoError(t, os.MkdirAll(baseDir, 0o750))
+	backend := startLockProbingS3Host(t, baseDir, volumeID)
+	cfg := &Config{
+		NodeName:               "node-a",
+		BaseDir:                baseDir,
+		S3Host:                 backend.url,
+		Bucket:                 "test-bucket",
+		Region:                 "us-east-1",
+		AccessKey:              "test-access-key",
+		SecretKey:              "test-secret-key",
+		loadStateRetryAttempts: 1,
+	}
+	installTestVolumeLeases(t, cfg, natsURL)
+	return cfg, dataDir, backend
+}
+
+// holdLockAsExport stands in for an nbdkit this node runs but never
+// registered, as a failed rebuild in recovery leaves behind.
+func holdLockAsExport(t *testing.T, cfg *Config, volumeID string) {
+	t.Helper()
+	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = viperblock.ReleaseVolumeLock(lock) })
+}
+
+// An unregistered export still takes guest writes, so the snapshot must have
+// it drain before an engine reads the backend's checkpoint.
+func TestSnapshotVolumeEngine_UnregisteredExportIsDrainedBeforeTheOpen(t *testing.T) {
+	const volumeID, snapshotID = "vol-strayserving", "snap-strayserving001"
+	cfg, dataDir, backend := unmountedSnapshotFixture(t, volumeID)
+	holdLockAsExport(t, cfg, volumeID)
+	accepted, release := testutil.StartBlockingDrainSocket(t, dataDir, volumeID, "OK\n")
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+		done <- err
+	}()
+
+	select {
+	case <-accepted:
+	case <-time.After(5 * time.Second):
+		t.Fatal("an unregistered export was never asked to drain")
+	}
+	assert.Zero(t, backend.requests.Load(), "the engine must not read the backend before the drain acks")
+	release()
+
+	select {
+	case err := <-done:
+		require.Error(t, err, "the refusing backend fails the open")
+		assert.NotErrorIs(t, err, errExportDrainFailed)
+	case <-time.After(10 * time.Second):
+		t.Fatal("snapshot did not proceed after the drain acked")
+	}
+	assert.Positive(t, backend.requests.Load(), "a drained export lets the engine open")
+}
+
+// An unregistered export with no socket cannot be drained, so the snapshot is
+// refused as retry-later before any engine reads the backend.
+func TestSnapshotVolumeEngine_UnregisteredExportWithoutSocketIsRetryLater(t *testing.T) {
+	const volumeID, snapshotID = "vol-straysettle", "snap-straysettle0001"
+	cfg, _, backend := unmountedSnapshotFixture(t, volumeID)
+	holdLockAsExport(t, cfg, volumeID)
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportSettling)
+	assert.Nil(t, snapshot)
+	assert.Equal(t, ebsprovider.ErrorCodeUnavailable, snapshotErrorCode(err))
+	assert.Zero(t, backend.requests.Load(), "a refused snapshot must not open an engine")
+}
+
+// A departed export that left unsealed WAL holds writes the backend lacks.
+func TestSnapshotVolumeEngine_UnmountedUnsealedWALFailsBeforeTheOpen(t *testing.T) {
+	const volumeID, snapshotID = "vol-strayunsealed", "snap-strayunsealed01"
+	cfg, _, backend := unmountedSnapshotFixture(t, volumeID)
+	leaveUnsealedWAL(t, cfg, volumeID)
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.Nil(t, snapshot)
+	assert.Zero(t, backend.requests.Load(), "a refused snapshot must not open an engine")
+}
+
+// With no export to drain, the volume lock is held across the open so a
+// writer cannot start between the check and the read.
+func TestSnapshotVolumeEngine_UnmountedOpenHoldsTheVolumeLock(t *testing.T) {
+	const volumeID, snapshotID = "vol-strayidle", "snap-strayidle000001"
+	cfg, _, backend := unmountedSnapshotFixture(t, volumeID)
+
+	_, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.Error(t, err, "the refusing backend fails the open")
+	assert.NotErrorIs(t, err, errExportDrainFailed)
+	assert.NotErrorIs(t, err, errExportSettling)
+	require.Positive(t, backend.requests.Load(), "a node with no export and no WAL opens the engine")
+	assert.Zero(t, backend.lockable.Load(), "a writer could take the volume lock while the engine was reading")
+	lock, err := viperblock.AcquireVolumeLock(cfg.BaseDir, volumeID)
+	require.NoError(t, err, "the snapshot must give the lock up when it is done")
+	require.NoError(t, viperblock.ReleaseVolumeLock(lock))
+}
+
+// writeSealReceiptAt leaves the receipt the plugin writes after a seal at sealedAt.
+func writeSealReceiptAt(t *testing.T, cfg *Config, volumeID string, sealedAt time.Time) {
+	t.Helper()
+	receipt, err := json.Marshal(sealReceipt{Volume: volumeID, PID: 1, SealedAt: sealedAt})
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(sealReceiptPath(cfg.BaseDir, volumeID), receipt, 0o600))
+}
+
+// touchWAL dates every entry of the volume's local WAL to at.
+func touchWAL(t *testing.T, cfg *Config, volumeID string, at time.Time) {
+	t.Helper()
+	dir, err := localWALDir(cfg, volumeID)
+	require.NoError(t, err)
+	require.NoError(t, filepath.WalkDir(dir, func(path string, _ fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		return os.Chtimes(path, at, at)
+	}))
+}
+
+// A receipt is proof only for the WAL that existed when it was written. WAL a
+// later client wrote and failed to seal is newer than it, so it is refused.
+func TestSnapshotVolumeEngine_SealReceiptOlderThanTheWALFails(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainstaleseal", "snap-drainstaleseal1"
+	cfg, _, vb := mountedSnapshotFixture(t, volumeID)
+	sealedAt := time.Now().Add(-time.Minute)
+	writeSealReceiptAt(t, cfg, volumeID, sealedAt)
+	leaveUnsealedWAL(t, cfg, volumeID)
+	require.NoError(t, os.WriteFile(filepath.Join(cfg.BaseDir, volumeID, "wal", "chunks", "wal.00000002.bin"), []byte("w"), 0o600))
+	touchWAL(t, cfg, volumeID, sealedAt.Add(time.Second))
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.ErrorIs(t, err, errExportDrainFailed)
+	assert.Nil(t, snapshot)
+	requireNoSnapshot(t, vb, snapshotID)
+}
+
+// A receipt written after every WAL change attests to all of it.
+func TestSnapshotVolumeEngine_SealReceiptNewerThanTheWALProceeds(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainfreshseal", "snap-drainfreshseal1"
+	cfg, _, _ := mountedSnapshotFixture(t, volumeID)
+	leaveUnsealedWAL(t, cfg, volumeID)
+	walChanged := time.Now().Add(-time.Minute)
+	touchWAL(t, cfg, volumeID, walChanged)
+	writeSealReceiptAt(t, cfg, volumeID, walChanged.Add(time.Second))
+
+	snapshot, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.NoError(t, err)
+	assert.Equal(t, ebsprovider.SnapshotStateCompleted, snapshot.State)
+}
+
+// The plugin dates its receipt to the millisecond, after the seal's own WAL
+// changes, so a WAL change inside the receipt's millisecond is not newer.
+func TestSnapshotVolumeEngine_SealReceiptInTheWALsMillisecondProceeds(t *testing.T) {
+	const volumeID, snapshotID = "vol-drainsamemilli", "snap-drainsamemilli1"
+	cfg, _, _ := mountedSnapshotFixture(t, volumeID)
+	leaveUnsealedWAL(t, cfg, volumeID)
+	sealedAt := time.Now().Add(-time.Minute).Truncate(time.Millisecond)
+	touchWAL(t, cfg, volumeID, sealedAt.Add(400*time.Microsecond))
+	writeSealReceiptAt(t, cfg, volumeID, sealedAt)
+
+	_, err := snapshotVolumeEngine(t.Context(), cfg, volumeID, snapshotID)
+
+	require.NoError(t, err)
+}

spinifex/providers/ebs/viperblock/viperblock.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"github.com/mulgadc/spinifex/spinifex/foundation/netaddr"
+	"io/fs"
 	"log/slog"
 	"net/http"
 	"os"
@@ -23,6 +24,7 @@ import (
 	"github.com/mulgadc/bluebottle/pkg/masterkey"
 	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/mulgadc/viperblock/viperblock"
 	"github.com/mulgadc/viperblock/viperblock/backends/s3"
@@ -459,12 +461,22 @@ func openVolumeVB(ctx context.Context, cfg *Config, volumeName string) (*viperbl
 	if err != nil {
 		return nil, nil, err
 	}
+	vb, err := openLeasedVolumeVB(ctx, cfg, volumeName, lease)
+	if err != nil {
+		cfg.releaseVolumeLease(ctx, lease)
+		return nil, nil, err
+	}
+	return vb, lease, nil
+}
 
+// openLeasedVolumeVB is openVolumeVB under a lease the caller already holds,
+// for a caller with a check to make between the lease and the open. The lease
+// stays the caller's to release, on failure too.
+func openLeasedVolumeVB(ctx context.Context, cfg *Config, volumeName string, lease *volumeLease) (*viperblock.VB, error) {
 	vbconfig, s3cfg := volumeVBConfig(cfg, volumeName)
 	vb, err := viperblock.New(&vbconfig, "s3", s3cfg)
 	if err != nil {
-		cfg.releaseVolumeLease(ctx, lease)
-		return nil, nil, fmt.Errorf("new viperblock: %w", err)
+		return nil, fmt.Errorf("new viperblock: %w", err)
 	}
 
 	// New starts the chunk uploader and WAL syncer, so a failure below must
@@ -473,22 +485,21 @@ func openVolumeVB(ctx context.Context, cfg *Config, volumeName string) (*viperbl
 	defer func() {
 		if !opened {
 			vb.Detach()
-			cfg.releaseVolumeLease(ctx, lease)
 		}
 	}()
 
 	if err := vb.Backend.InitCtx(ctx); err != nil {
-		return nil, nil, fmt.Errorf("backend init: %w", err)
+		return nil, fmt.Errorf("backend init: %w", err)
 	}
 	if err := loadStateWithRetry(ctx, cfg, vb, volumeName); err != nil {
 		if errors.Is(err, viperblock.ErrStateNotFound) {
 			cfg.clearVolumeDirty(ctx, volumeName, lease.generation)
 		}
-		return nil, nil, fmt.Errorf("load state: %w", err)
+		return nil, fmt.Errorf("load state: %w", err)
 	}
 
 	opened = true
-	return vb, lease, nil
+	return vb, nil
 }
 
 // isAuxVolume reports whether a volume is an -efi auxiliary volume. Auxiliary
@@ -546,6 +557,55 @@ func volumeNeedsSeal(volumeName, baseDir string) bool {
 	return err == nil
 }
 
+// volumeHoldsLocalWAL reports whether an nbdkit plugin has opened the
+// volume's WAL here and not removed it. A successful seal removes it; an
+// export no client has connected to yet never created it.
+func volumeHoldsLocalWAL(cfg *Config, volumeName string) bool {
+	dir, err := localWALDir(cfg, volumeName)
+	if err != nil {
+		return false
+	}
+	_, err = os.Stat(dir)
+	return err == nil
+}
+
+// localWALDir is where the nbdkit plugin keeps volumeName's WAL on this node.
+func localWALDir(cfg *Config, volumeName string) (string, error) {
+	walBase := cfg.WALBaseDir
+	if walBase == "" {
+		walBase = cfg.BaseDir
+	}
+	dir, err := localVolumeDir(walBase, volumeName)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, "wal"), nil
+}
+
+// latestWALChange is the newest modification time under the volume's local
+// WAL, the directory itself included, so a file created or removed counts too.
+func latestWALChange(cfg *Config, volumeName string) (time.Time, error) {
+	dir, err := localWALDir(cfg, volumeName)
+	if err != nil {
+		return time.Time{}, err
+	}
+	var latest time.Time
+	err = filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+		if info.ModTime().After(latest) {
+			latest = info.ModTime()
+		}
+		return nil
+	})
+	return latest, err
+}
+
 // sealReceiptSuffix names the file the nbdkit plugin leaves at
 // baseDir/<volume>.sealed after a successful seal.
 const sealReceiptSuffix = ".sealed"
@@ -584,19 +644,53 @@ func consumeSealReceipt(baseDir, volume string) bool {
 	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
 		slog.Warn("failed to remove seal receipt", "volume", volume, "path", path, "err", err)
 	}
+	_, ok := validSealReceipt(data, volume, path)
+	return ok
+}
 
-	var receipt sealReceipt
-	if err := json.Unmarshal(data, &receipt); err != nil {
-		slog.Warn("malformed seal receipt", "volume", volume, "path", path, "err", err)
+// sealReceiptCoversWAL reports whether the volume's seal receipt postdates
+// every change to its local WAL, leaving the receipt for the unmount it
+// attests to. An older receipt is an earlier client's, not the last one's.
+func sealReceiptCoversWAL(cfg *Config, volume string) bool {
+	path := sealReceiptPath(cfg.BaseDir, volume)
+	data, err := os.ReadFile(path)
+	if err != nil {
 		return false
 	}
-	if receipt.Volume != volume || receipt.SealedAt.IsZero() {
-		slog.Warn("seal receipt missing required fields", "volume", volume, "path", path, "receipt", receipt)
+	receipt, ok := validSealReceipt(data, volume, path)
+	if !ok {
+		return false
+	}
+	latest, err := latestWALChange(cfg, volume)
+	if err != nil {
+		slog.Warn("could not read local WAL to date its seal receipt", "volume", volume, "err", err)
+		return false
+	}
+	// The plugin writes SealedAt to the millisecond, after the seal's own WAL
+	// changes, so a WAL change in the receipt's millisecond is the seal's.
+	if receipt.SealedAt.Before(latest.Truncate(time.Millisecond)) {
+		slog.Warn("seal receipt predates the local WAL, treating the WAL as unsealed",
+			"volume", volume, "sealed_at", receipt.SealedAt, "wal_changed_at", latest)
 		return false
 	}
 	return true
 }
 
+// validSealReceipt decodes data, reporting whether it is a receipt the plugin
+// wrote for volume.
+func validSealReceipt(data []byte, volume, path string) (sealReceipt, bool) {
+	var receipt sealReceipt
+	if err := json.Unmarshal(data, &receipt); err != nil {
+		slog.Warn("malformed seal receipt", "volume", volume, "path", path, "err", err)
+		return sealReceipt{}, false
+	}
+	if receipt.Volume != volume || receipt.SealedAt.IsZero() {
+		slog.Warn("seal receipt missing required fields", "volume", volume, "path", path, "receipt", receipt)
+		return sealReceipt{}, false
+	}
+	return receipt, true
+}
+
 // clearStaleSealReceipt removes any seal receipt left by a previous mount of
 // this volume. Staleness is handled here, at mount, rather than by matching
 // PIDs at unmount: once this runs, any receipt found at the next unmount can
@@ -733,6 +827,9 @@ func (cfg *Config) RecoverMountedVolumes(ctx context.Context, nc *nats.Conn) {
 // readiness: without the round trip a caller can act on the signal and still
 // have its request arrive before the interest does.
 func (cfg *Config) Run(nc *nats.Conn) error {
+	if err := cfg.validateNodeName(); err != nil {
+		return err
+	}
 	if err := cfg.LoadEncryptionKey(); err != nil {
 		return err
 	}
@@ -746,11 +843,7 @@ func (cfg *Config) Run(nc *nats.Conn) error {
 
 	cfg.RecoverMountedVolumes(context.Background(), nc)
 
-	if cfg.NodeName != "" {
-		slog.Info("Waiting for EBS events", "node", cfg.NodeName)
-	} else {
-		slog.Info("Waiting for EBS events (single-node mode)")
-	}
+	slog.Info("Waiting for EBS events", "node", cfg.NodeName)
 
 	if err := RegisterVolumeLifecycleSubjects(cfg, nc); err != nil {
 		return err
@@ -768,6 +861,15 @@ func (cfg *Config) Run(nc *nats.Conn) error {
 	return nil
 }
 
+// validateNodeName refuses a node name the contract cannot address. Without
+// one the fence notice has no subject, and a fenced guest would keep running.
+func (cfg *Config) validateNodeName() error {
+	if err := ebsprovider.ValidateNodeID(cfg.NodeName); err != nil {
+		return fmt.Errorf("viperblockd node name %q (cluster config node): %w", cfg.NodeName, err)
+	}
+	return nil
+}
+
 // Shutdown flushes each mounted volume's WAL and reaps idle nbdkit processes.
 // It snapshots MountedVolumes and clears it while holding the lock, then
 // flushes/kills outside the lock since VB.Close does heavy I/O.

spinifex/providers/ebs/viperblock/viperblockd_integration_test.go

@@ -839,42 +839,6 @@ func TestIntegration_ServiceGracefulShutdown(t *testing.T) {
 	t.Log("Service is running and responsive")
 }
 
-// TestIntegration_GenericTopicRouting tests that NodeName="" uses generic ebs.unmount with queue group.
-func TestIntegration_GenericTopicRouting(t *testing.T) {
-	t.Parallel()
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	ns, natsURL := setupEmbeddedNATS(t)
-	defer ns.Shutdown()
-
-	cfg := setupTestConfig(t, natsURL)
-	cfg.NodeName = "" // Single-node / generic mode
-	cfg.MountedVolumes = []MountedVolume{
-		{Name: "vol-generic", PID: deadpid.PID},
-	}
-
-	startTestService(t, cfg)
-
-	nc, err := nats.Connect(natsURL)
-	assert.NoError(t, err)
-	defer nc.Close()
-
-	// Send to generic topic (not ebs.{node}.unmount)
-	request := viperblocklegacyv1.EBSRequest{Name: "vol-generic"}
-	requestData, _ := json.Marshal(request)
-
-	msg, err := nc.Request("ebs.unmount", requestData, 3*time.Second)
-	assert.NoError(t, err)
-
-	var response viperblocklegacyv1.EBSUnMountResponse
-	assert.NoError(t, json.Unmarshal(msg.Data, &response))
-	assert.Equal(t, "vol-generic", response.Volume)
-	assert.False(t, response.Mounted)
-	assert.Empty(t, response.Error)
-}
-
 // TestIntegration_MountAuxiliaryVolumeSuffix tests that volumes with -cloudinit suffix
 // reach the mount handler and trigger the auxiliary volume code path.
 func TestIntegration_MountAuxiliaryVolumeSuffix(t *testing.T) {

spinifex/providers/ebs/viperblock/volume_fence.go

@@ -9,8 +9,8 @@ import (
 	"os"
 	"time"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -319,46 +319,40 @@ func (cfg *Config) fenceNoticeRetry() time.Duration {
 }
 
 // publishVolumeFenced tells the local daemon a guest is running against a disk
-// that is gone, until the daemon acknowledges. It gives up on a closed
-// connection, or once the volume is exported here again by a newer mount.
+// that is gone, until the daemon acknowledges. It gives up on a notice that can
+// never be acted on, a closed connection, or the volume being exported again.
 func (cfg *Config) publishVolumeFenced(ctx context.Context, volumeName, winner string) {
 	if cfg.nc == nil {
 		return
 	}
-	event := viperblocklegacyv1.VolumeFencedEvent{
-		Volume: volumeName,
-		Node:   cfg.leaseOwner(),
-		Winner: winner,
-		Reason: fmt.Sprintf("volume lease moved to %s while this node had the volume open", winner),
+	notice := ebsprovider.FenceNotice{
+		Versioned: ebsprovider.NewVersioned(),
+		VolumeID:  volumeName,
+		NodeID:    cfg.NodeName,
+		Winner:    winner,
+		Reason:    fmt.Sprintf("volume lease moved to %s while this node had the volume open", winner),
 	}
-	payload, err := json.Marshal(event)
-	if err != nil {
-		slog.ErrorContext(ctx, "fence: marshal fenced event", "volume", volumeName, "err", err)
-		return
-	}
-	subject := viperblocklegacyv1.VolumeFencedSubject(cfg.NodeName)
 	began := time.Now()
 
 	for attempt := 1; ; attempt++ {
-		reqCtx, cancel := context.WithTimeout(ctx, fenceNoticeTimeout)
-		reply, err := cfg.nc.RequestWithContext(reqCtx, subject, payload)
-		cancel()
+		instanceID, err := ebsprovider.NotifyFenced(ctx, cfg.nc, notice, fenceNoticeTimeout)
 		if err == nil {
-			var ack viperblocklegacyv1.VolumeFencedAck
-			if jerr := json.Unmarshal(reply.Data, &ack); jerr == nil && ack.Error != "" {
-				slog.ErrorContext(ctx, "fence: the daemon could not act on the fenced volume",
-					"volume", volumeName, "subject", subject, "err", ack.Error)
-			} else if attempt > 1 {
+			if attempt > 1 {
 				slog.WarnContext(ctx, "fence: the daemon acknowledged the fenced volume after retrying",
-					"volume", volumeName, "instance", ack.Instance, "attempts", attempt,
+					"volume", volumeName, "instance", instanceID, "attempts", attempt,
 					"unacknowledged_ms", otelsetup.Millis(time.Since(began)))
 			}
 			return
 		}
+		if !ebsprovider.FenceNoticeRetryable(err) {
+			slog.ErrorContext(ctx, "fence: the daemon cannot act on the fenced volume, the guest is left with dead I/O",
+				"volume", volumeName, "node", notice.NodeID, "attempts", attempt, "err", err)
+			return
+		}
 
 		if cfg.nc.IsClosed() {
 			slog.ErrorContext(ctx, "fence: connection closed before the daemon acknowledged the fenced volume, the guest may be left with dead I/O",
-				"volume", volumeName, "subject", subject, "attempts", attempt)
+				"volume", volumeName, "node", notice.NodeID, "attempts", attempt)
 			return
 		}
 		if _, again := cfg.lookupMountedVolume(volumeName); again {
@@ -368,7 +362,7 @@ func (cfg *Config) publishVolumeFenced(ctx context.Context, volumeName, winner s
 		}
 		if attempt == 1 || attempt%fenceNoticeLogEveryTries == 0 {
 			slog.ErrorContext(ctx, "fence: the daemon has not acknowledged the fenced volume, retrying; until it does the guest keeps running with dead I/O",
-				"volume", volumeName, "subject", subject, "attempts", attempt,
+				"volume", volumeName, "node", notice.NodeID, "attempts", attempt,
 				"unacknowledged_ms", otelsetup.Millis(time.Since(began)), "err", err)
 		}
 

spinifex/providers/ebs/viperblock/volume_fence_invariants_test.go

@@ -11,7 +11,7 @@ import (
 	"testing"
 	"time"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -133,16 +133,16 @@ func TestS4_FenceHolderReadIsBounded(t *testing.T) {
 // fenceResponder answers fence notices as the daemon does and records them.
 type fenceResponder struct {
 	mu     sync.Mutex
-	events []viperblocklegacyv1.VolumeFencedEvent
+	events []ebsprovider.FenceNotice
 }
 
 func (r *fenceResponder) handle(msg *nats.Msg) {
-	var event viperblocklegacyv1.VolumeFencedEvent
-	_ = json.Unmarshal(msg.Data, &event)
+	var notice ebsprovider.FenceNotice
+	_ = json.Unmarshal(msg.Data, &notice)
 	r.mu.Lock()
-	r.events = append(r.events, event)
+	r.events = append(r.events, notice)
 	r.mu.Unlock()
-	payload, _ := json.Marshal(viperblocklegacyv1.VolumeFencedAck{Instance: "i-fenced"})
+	payload, _ := json.Marshal(ebsprovider.FenceNoticeAck{Versioned: ebsprovider.ReplyVersioned(notice.SchemaVersion), InstanceID: "i-fenced"})
 	_ = msg.Respond(payload)
 }
 
@@ -185,7 +185,9 @@ func TestS4_FenceNoticeSurvivesAMissedDelivery(t *testing.T) {
 	require.NoError(t, err)
 	t.Cleanup(daemonConn.Close)
 	responder := &fenceResponder{}
-	_, err = daemonConn.Subscribe(viperblocklegacyv1.VolumeFencedSubject("node-a"), responder.handle)
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	_, err = daemonConn.Subscribe(subject, responder.handle)
 	require.NoError(t, err)
 	require.NoError(t, daemonConn.Flush())
 
@@ -197,7 +199,10 @@ func TestS4_FenceNoticeSurvivesAMissedDelivery(t *testing.T) {
 	require.GreaterOrEqualf(t, responder.count(), 1,
 		"ADR-0007:S4: a daemon that subscribed after the first notice never heard the fence, so its guest runs on with no disk. "+
 			"Fix: retry publishVolumeFenced (volume_fence.go) until the daemon acknowledges")
-	assert.Equal(t, volumeName, responder.events[0].Volume)
+	first := responder.events[0]
+	assert.Equal(t, volumeName, first.VolumeID)
+	assert.Equal(t, "node-a", first.NodeID)
+	assert.True(t, ebsprovider.SupportsSchemaVersion(first.SchemaVersion), "the daemon refuses a notice in a version it cannot read")
 
 	seen := responder.count()
 	time.Sleep(5 * cfg.fenceNoticeEvery)
@@ -252,3 +257,93 @@ func TestS4_FenceNoticeStopsWithTheConnection(t *testing.T) {
 		t.Fatal("a fence notice kept retrying on a closed connection")
 	}
 }
+
+// TestFenceNotice_UnavailableDaemonIsAskedAgain keeps a refusal that says
+// "not now" from ending delivery: the guest is still running on a dead disk.
+func TestFenceNotice_UnavailableDaemonIsAskedAgain(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencenoticeunavailable"
+	cfg, nc := noticeConfig(t, natsURL, volumeName)
+
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	unavailable, err := json.Marshal(ebsprovider.FenceNoticeAck{
+		Versioned: ebsprovider.NewVersioned(),
+		Error:     &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "not yet"},
+	})
+	require.NoError(t, err)
+	ack, err := json.Marshal(ebsprovider.FenceNoticeAck{Versioned: ebsprovider.NewVersioned(), InstanceID: "i-fenced"})
+	require.NoError(t, err)
+	var heard atomic.Int32
+	_, err = nc.Subscribe(subject, func(msg *nats.Msg) {
+		if heard.Add(1) == 1 {
+			_ = msg.Respond(unavailable)
+			return
+		}
+		_ = msg.Respond(ack)
+	})
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.onVolumeLeaseLost(context.Background(), volumeName, leaseLostStalled) }()
+	select {
+	case <-done:
+	case <-time.After(fenceNoticeTimeout + 2*time.Second):
+		t.Fatal("a fence notice was still being sent after the daemon acknowledged it")
+	}
+	assert.Equal(t, int32(2), heard.Load(), "ADR-0007:S4: a daemon that was unavailable was never asked again")
+}
+
+// TestFenceNotice_UnsendableNoticeIsNotRetried stops a notice that can never
+// be addressed from spinning until shutdown on a goroutine nobody watches.
+func TestFenceNotice_UnsendableNoticeIsNotRetried(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencenoticeunsendable"
+	cfg, _ := noticeConfig(t, natsURL, volumeName)
+	cfg.NodeName = ""
+	cfg.MountedVolumes = nil // gone, as after the teardown, so only the error can end the retry
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.publishVolumeFenced(t.Context(), volumeName, "node-b") }()
+	select {
+	case <-done:
+	case <-time.After(fenceNoticeTimeout + 2*time.Second):
+		t.Fatal("a fence notice with no subject to send it on kept retrying")
+	}
+}
+
+// TestFenceNotice_RefusedNoticeIsNotRepeated keeps a notice the daemon heard and
+// could not act on from being resent forever: a refusal is an answer.
+func TestFenceNotice_RefusedNoticeIsNotRepeated(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencenoticerefused"
+	cfg, nc := noticeConfig(t, natsURL, volumeName)
+
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	var heard atomic.Int32
+	refusal, err := json.Marshal(ebsprovider.FenceNoticeAck{
+		Versioned: ebsprovider.NewVersioned(),
+		Error:     &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: "no"},
+	})
+	require.NoError(t, err)
+	_, err = nc.Subscribe(subject, func(msg *nats.Msg) {
+		heard.Add(1)
+		_ = msg.Respond(refusal)
+	})
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.onVolumeLeaseLost(context.Background(), volumeName, leaseLostStalled) }()
+	select {
+	case <-done:
+	case <-time.After(fenceNoticeTimeout + 2*time.Second):
+		t.Fatal("a fence notice kept retrying after the daemon refused it")
+	}
+	assert.Equal(t, int32(1), heard.Load())
+}

spinifex/providers/ebs/viperblock/volume_fence_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -171,15 +172,25 @@ func TestVolumeFence_CompletesOnceTheWriterFinallyExits(t *testing.T) {
 
 	const volumeName = "vol-fencekilllate"
 
-	cfg := fencedConfig(t, natsURL, "node-a", volumeName)
+	cfg, nc := noticeConfig(t, natsURL, volumeName)
 	cfg.MountedVolumes[0].PID = 0 // rejected by ForceKillProcess, so the kill cannot succeed
 	// Alive for the guard and one watch pass, gone on the next: the fence has to
 	// see the failure persist and then resolve.
 	var probes atomic.Int32
 	cfg.processAlive = func(int) bool { return probes.Add(1) <= 2 }
 	cfg.fenceWatchEvery = 10 * time.Millisecond
+	responder := &fenceResponder{}
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	_, err = nc.Subscribe(subject, responder.handle)
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
 
 	cfg.fenceVolume(t.Context(), volumeName, leaseLostToPeer)
+	require.Equalf(t, 1, responder.count(),
+		"ADR-0007:S4: a fence that completed late never told the daemon, so its guest runs on with no disk. "+
+			"Fix: publishVolumeFenced once awaitFencedWriterExit sees the writer gone (volume_fence.go)")
+	assert.Equal(t, volumeName, responder.events[0].VolumeID)
 
 	cfg.mu.Lock()
 	mounted := len(cfg.MountedVolumes)

spinifex/runtime/compute/vm/desired_state_test.go

@@ -107,7 +107,7 @@ func TestVMMarshal_LegacyHostfwdNotWrittenBack(t *testing.T) {
 	assert.Contains(t, raw, "hostfwd_port_map")
 }
 
-// The other twelve command booleans had nowhere to go because nothing read them
+// The other eleven command booleans had nowhere to go because nothing read them
 // back. A record must carry none of them, so they are gone rather than unread.
 func TestVMMarshal_CommandBooleansAreNotPersisted(t *testing.T) {
 	legacy, err := json.Marshal(map[string]any{
@@ -115,7 +115,6 @@ func TestVMMarshal_CommandBooleansAreNotPersisted(t *testing.T) {
 		"attributes": ec2v1.EC2CommandAttributes{
 			AttachVolume:                true,
 			DetachVolume:                true,
-			DrainVolume:                 true,
 			StartInstance:               true,
 			RebootInstance:              true,
 			AttachENI:                   true,
@@ -137,7 +136,7 @@ func TestVMMarshal_CommandBooleansAreNotPersisted(t *testing.T) {
 	require.NoError(t, err)
 
 	for _, field := range []string{
-		"attach_volume", "detach_volume", "drain_volume", "start_instance",
+		"attach_volume", "detach_volume", "start_instance",
 		"reboot_instance", "attach_eni", "detach_eni",
 		"associate_iam_instance_profile", "set_spot_lineage", "set_instance_tags",
 		"remove_instance_tags", "set_instance_monitoring",

spinifex/runtime/compute/vm/errors.go

@@ -32,11 +32,10 @@ var ErrVolumeNotDetachable = errors.New("volume cannot be detached")
 // supplies a device name that does not match the recorded attachment.
 var ErrVolumeDeviceMismatch = errors.New("volume device name mismatch")
 
-// ErrMountAmbiguous is returned by VolumeMounter.MountOne when the
-// backend acknowledged the mount but returned an empty URI. The mount may
-// or may not have started serving NBD; AttachVolume must invoke UnmountOne
-// defensively so a half-started backend mount is not orphaned.
-var ErrMountAmbiguous = errors.New("ebs.mount succeeded with empty URI")
+// ErrMountAmbiguous is returned by VolumeMounter.Mount and MountOne when a sent
+// publish got no usable export back (bad reply, timeout, cancelled caller), so
+// the provider may have started one; it is unpublished so it is not orphaned.
+var ErrMountAmbiguous = errors.New("publish reply carried no usable export")
 
 // ErrENINotAttached is returned by HotUnplugENI when the supplied ENI id
 // is not present in the instance's ENIRequests.AttachedByENIID map.

spinifex/runtime/compute/vm/firmware.go

@@ -1,21 +1,14 @@
 package vm
 
 import (
-	"crypto/sha256"
-	"encoding/hex"
 	"fmt"
-	"io"
 	"os"
-	"sync"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 )
 
 // FirmwareCandidate is a (CODE, VARS template) pair shipped together by a
-// firmware package. CODE is mounted readonly as pflash unit 0; VARS seeds the
-// instance's variable store, which is mounted as pflash unit 1 so EFI
-// variables (BootOrder, BootNext, secure-boot state) survive reboots.
+// firmware package. CODE is mounted readonly as pflash unit 0; VARS is copied
+// into a per-VM viperblock volume on first boot and mounted as pflash unit 1
+// so EFI variables (BootOrder, BootNext, secure-boot state) survive reboots.
 type FirmwareCandidate struct {
 	Code         string
 	VarsTemplate string
@@ -55,67 +48,3 @@ func FirmwarePaths(arch string) (code, varsTemplate string, varsSize int64, err
 	}
 	return "", "", 0, fmt.Errorf("no UEFI firmware found for architecture %q; install ovmf (x86_64) or qemu-efi-aarch64 (arm64)", arch)
 }
-
-// NodeFirmware is the UEFI firmware this node runs for one architecture: the
-// candidate FirmwarePaths matched and the identity a variable store is
-// checked against.
-type NodeFirmware struct {
-	Code         string
-	VarsTemplate string
-	Identity     uefistore.FirmwareIdentity
-}
-
-// codeHashKey identifies one version of a CODE blob on disk, so a package
-// update that replaces the file is hashed again.
-type codeHashKey struct {
-	path    string
-	size    int64
-	modTime time.Time
-}
-
-// codeHashes caches CODE blob hashes by codeHashKey. The arm64 blob is 64 MiB,
-// and every UEFI launch needs its hash.
-var codeHashes sync.Map
-
-// NodeFirmwareFor returns the firmware this node runs for arch, with the
-// SHA-256 of its CODE blob.
-func NodeFirmwareFor(arch string) (NodeFirmware, error) {
-	code, vars, varsSize, err := FirmwarePaths(arch)
-	if err != nil {
-		return NodeFirmware{}, err
-	}
-	sum, err := codeSHA256(code)
-	if err != nil {
-		return NodeFirmware{}, err
-	}
-	return NodeFirmware{
-		Code:         code,
-		VarsTemplate: vars,
-		Identity:     uefistore.FirmwareIdentity{Arch: arch, VarsSize: varsSize, CodeSHA256: sum},
-	}, nil
-}
-
-func codeSHA256(path string) (string, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return "", fmt.Errorf("open UEFI CODE %s: %w", path, err)
-	}
-	defer f.Close()
-	info, err := f.Stat()
-	if err != nil {
-		return "", fmt.Errorf("stat UEFI CODE %s: %w", path, err)
-	}
-	key := codeHashKey{path: path, size: info.Size(), modTime: info.ModTime()}
-	if cached, ok := codeHashes.Load(key); ok {
-		if sum, ok := cached.(string); ok {
-			return sum, nil
-		}
-	}
-	h := sha256.New()
-	if _, err := io.Copy(h, f); err != nil {
-		return "", fmt.Errorf("hash UEFI CODE %s: %w", path, err)
-	}
-	sum := hex.EncodeToString(h.Sum(nil))
-	codeHashes.Store(key, sum)
-	return sum, nil
-}

spinifex/runtime/compute/vm/lifecycle.go

@@ -92,9 +92,6 @@ func (m *Manager) Start(ctx context.Context, id string) error {
 	if !ok {
 		return fmt.Errorf("%w: %s", ErrInstanceNotFound, id)
 	}
-	if err := CheckUEFIVarStore(instance); err != nil {
-		return err
-	}
 	// StateError is rejected by launchStillValid; move to Pending first so
 	// launch proceeds (resource re-allocation is the caller's responsibility).
 	if m.Status(instance) == StateError && m.deps.TransitionState != nil {
@@ -130,13 +127,6 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
 		return nil
 	}
 
-	// Before the reason below is cleared, so a refused start changes nothing.
-	if err := CheckUEFIVarStore(instance); err != nil {
-		slog.ErrorContext(ctx, "Refusing to launch a UEFI instance without exactly one variable store",
-			"instanceId", instance.ID, "err", err)
-		return err
-	}
-
 	// A reason left by the stop, or by a restore that found this node could not
 	// schedule the instance, describes a state the instance is now leaving.
 	// Clearing it here keeps the node the only writer of the field.
@@ -158,16 +148,6 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
 		}
 	}
 
-	// Before the mount, so a store this node cannot boot fails the launch
-	// with nothing to roll back.
-	uefiCtx, uefiSpan := otel.Tracer(vmTracerName).Start(ctx, "vm.launch.uefi_var_store")
-	uefiErr := m.prepareUEFIVarStore(uefiCtx, instance)
-	endSpanWithError(uefiSpan, uefiErr)
-	if uefiErr != nil {
-		slog.ErrorContext(ctx, "Failed to prepare the UEFI variable store", "instanceId", instance.ID, "err", uefiErr)
-		return uefiErr
-	}
-
 	_, mountSpan := otel.Tracer(vmTracerName).Start(ctx, "vm.launch.mount_volumes")
 	mountErr := m.deps.VolumeMounter.Mount(ctx, instance)
 	endSpanWithError(mountSpan, mountErr)
@@ -344,14 +324,6 @@ func (m *Manager) startQEMU(instance *VM, appendConsole bool) error {
 		instance.Config.IOThreads = append(instance.Config.IOThreads, driveCfg.IOThreads...)
 		instance.Config.Devices = append(instance.Config.Devices, driveCfg.Devices...)
 		instance.Config.Blockdevs = append(instance.Config.Blockdevs, driveCfg.Blockdevs...)
-
-		varsDrive, err := m.uefiVarsDrive(instance)
-		if err != nil {
-			return err
-		}
-		if varsDrive != nil {
-			instance.Config.Drives = append(instance.Config.Drives, *varsDrive)
-		}
 	}
 
 	if instance.DirectBoot {
@@ -1350,7 +1322,7 @@ func buildBaseVMConfig(instanceID, instanceType, pidFile, consoleLogPath, serial
 		CPUCount:       vCPUs,
 		Architecture:   architecture,
 		InstanceType:   instanceType,
-		UseUEFI:        isUEFIBootMode(bootMode),
+		UseUEFI:        bootMode == "uefi" || bootMode == "uefi-preferred",
 
 		// Present EC2-shaped DMI so stock cloud-init activates the Ec2 datasource.
 		SMBIOSUUID:         ec2SMBIOSUUID(instanceID),
@@ -1443,8 +1415,7 @@ func buildDiskBootDrives(requests []EBSRequest, cpuCount int, machineType string
 //
 // The boot volume is a -blockdev node named "os" under a bootindex=1 device
 // that carries werror/rerror; it is never detached, so it keeps fixed names.
-// EFI volumes emit pflash unit=1, as uefiVarsDrive does for a variable store
-// pointer; Config.Execute adds the CODE blob (unit=0).
+// EFI volumes emit pflash unit=1; Config.Execute adds the CODE blob (unit=0).
 //
 // Every other (data) volume is given the same named node-name/device-id/
 // iothread-id shape AttachVolume's hot-plug path uses, via a -blockdev entry

spinifex/runtime/compute/vm/manager.go

@@ -87,9 +87,6 @@ type Deps struct {
 	// (predastore + viperblock) is confirmed up. Nil is treated as ready,
 	// preserving behaviour for tests and nodes with storage unwired.
 	BackingStoreReady func() bool
-
-	// UEFIVarStore is where UEFI variable stores are read from and kept.
-	UEFIVarStore UEFIVarStoreDeps
 }
 
 // Manager owns the in-memory map of running VMs on this node and every

spinifex/runtime/compute/vm/uefi_varstore.go

@@ -1,390 +0,0 @@
-package vm
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log/slog"
-	"os"
-	"path/filepath"
-	"slices"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
-	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/metric"
-)
-
-const (
-	// uefiStoreDirName is the directory under DataDir that holds one
-	// subdirectory per instance with its live variable store.
-	uefiStoreDirName   = "uefi"
-	uefiVarsFileName   = "vars.fd"
-	uefiSeededFileName = "seeded.json"
-
-	// uefiStoreGetTimeout bounds the one get a launch makes for a fixed-size
-	// object, so a wedged object store fails the start rather than holding it.
-	uefiStoreGetTimeout = 10 * time.Second
-
-	// uefiStoreBodySlack is how far a compressed body may exceed its region
-	// before it is refused; gzip grows incompressible data by far less.
-	uefiStoreBodySlack = 1 << 20
-
-	// uefiStateReasonCode is the StateReason a launch refused over its
-	// variable store records, as EC2 reports an internal launch failure.
-	uefiStateReasonCode = "Server.InternalError"
-)
-
-// ErrUEFIVarStoreMissing is a UEFI instance whose record names no variable
-// store and carries no EFI volume, so it has nothing to boot its variables from.
-var ErrUEFIVarStoreMissing = errors.New("UEFI instance has no variable store")
-
-// ErrUEFIVarStoreConflict is a UEFI instance whose record names a variable
-// store and also carries an EFI volume, so either one would be ignored.
-var ErrUEFIVarStoreConflict = errors.New("UEFI instance has both a variable store pointer and an EFI volume")
-
-// ErrUEFIFirmwareMismatch is a variable store written under firmware whose
-// architecture or VARS size differs from this node's, which pflash cannot load.
-var ErrUEFIFirmwareMismatch = errors.New("UEFI variable store firmware does not match this node")
-
-// ErrUEFIVarStoreUnusable is a variable store object that is missing or does
-// not decode to a region of this node's VARS size. Retrying does not help.
-var ErrUEFIVarStoreUnusable = errors.New("UEFI variable store object is unusable")
-
-// UEFIVarStoreDeps is where a node reads instance variable stores from and
-// keeps the live copy QEMU opens. DataDir is the daemon's persistent data
-// directory, so the copy survives a daemon restart and a host reboot.
-type UEFIVarStoreDeps struct {
-	Objects objectstore.ObjectStore
-	Bucket  string
-	DataDir string
-}
-
-// uefiSeed is seeded.json: the pointer key vars.fd beside it was seeded from.
-type uefiSeed struct {
-	Key      string    `json:"key"`
-	SeededAt time.Time `json:"seeded_at"`
-}
-
-// isUEFIBootMode reports whether bootMode boots the guest through UEFI firmware.
-func isUEFIBootMode(bootMode string) bool {
-	return bootMode == "uefi" || bootMode == "uefi-preferred"
-}
-
-// usesUEFIVarStore reports whether v boots through UEFI and so needs a
-// variable store. A direct-boot microvm boots qboot and has none.
-func usesUEFIVarStore(v *VM) bool {
-	return !v.DirectBoot && isUEFIBootMode(v.BootMode)
-}
-
-// CheckUEFIVarStore refuses a UEFI instance that has no variable store, or
-// two, before anything about it is changed. An EFI volume still counts as a
-// store until RunInstances seeds pointers instead. The caller owns v.
-func CheckUEFIVarStore(v *VM) error {
-	if !usesUEFIVarStore(v) {
-		return nil
-	}
-	v.EBSRequests.Mu.Lock()
-	hasEFIVolume := slices.ContainsFunc(v.EBSRequests.Requests, func(r EBSRequest) bool { return r.EFI })
-	v.EBSRequests.Mu.Unlock()
-
-	switch {
-	case v.UEFIVarStore == nil && !hasEFIVolume:
-		return fmt.Errorf("%w: %s", ErrUEFIVarStoreMissing, v.ID)
-	case v.UEFIVarStore != nil && hasEFIVolume:
-		return fmt.Errorf("%w: %s points at %s", ErrUEFIVarStoreConflict, v.ID, v.UEFIVarStore.Key)
-	}
-	return nil
-}
-
-// uefiInstanceDir is the directory holding instanceID's vars.fd and
-// seeded.json. An ID that is not a single path segment is refused.
-func uefiInstanceDir(dataDir, instanceID string) (string, error) {
-	if !strings.HasPrefix(instanceID, "i-") || strings.ContainsAny(instanceID, `/\`) || instanceID != filepath.Clean(instanceID) {
-		return "", fmt.Errorf("instance ID %q cannot name a UEFI store directory", instanceID)
-	}
-	return filepath.Join(dataDir, uefiStoreDirName, instanceID), nil
-}
-
-// uefiVarsPath is the live variable store QEMU opens as pflash unit 1.
-func (m *Manager) uefiVarsPath(instanceID string) (string, error) {
-	dir, err := uefiInstanceDir(m.deps.UEFIVarStore.DataDir, instanceID)
-	if err != nil {
-		return "", err
-	}
-	return filepath.Join(dir, uefiVarsFileName), nil
-}
-
-// uefiVarsDrive is pflash unit 1 from this node's copy of the store the
-// record points at, or nil with no pointer. Launch seeds the copy and a
-// reboot reuses it, so a missing one fails rather than booting without it.
-func (m *Manager) uefiVarsDrive(instance *VM) (*Drive, error) {
-	hasPointer := false
-	m.Inspect(instance, func(v *VM) { hasPointer = v.UEFIVarStore != nil })
-	if !hasPointer || !usesUEFIVarStore(instance) {
-		return nil, nil
-	}
-	path, err := m.uefiVarsPath(instance.ID)
-	if err != nil {
-		return nil, err
-	}
-	if _, err := os.Stat(path); err != nil {
-		return nil, fmt.Errorf("UEFI variable store for %s: %w", instance.ID, err)
-	}
-	return &Drive{File: path, Format: "raw", If: "pflash", Unit: 1}, nil
-}
-
-// prepareUEFIVarStore leaves vars.fd holding the store the record points at,
-// reusing this node's copy when seeded.json names the pointer and fetching
-// the object otherwise. The object and the record's pointer are only read.
-func (m *Manager) prepareUEFIVarStore(ctx context.Context, instance *VM) error {
-	var ref *UEFIVarStoreRef
-	m.Inspect(instance, func(v *VM) {
-		if v.UEFIVarStore != nil {
-			cp := *v.UEFIVarStore
-			ref = &cp
-		}
-	})
-	if ref == nil || !usesUEFIVarStore(instance) {
-		return nil
-	}
-
-	deps := m.deps.UEFIVarStore
-	if deps.Objects == nil || deps.Bucket == "" || deps.DataDir == "" {
-		return errors.New("UEFI variable store access is not wired on this node")
-	}
-	if name, ok := strings.CutPrefix(ref.Key, uefistore.KeyPrefix(instance.ID)); !ok || name == "" || strings.Contains(name, "/") {
-		return fmt.Errorf("%w: pointer %q is not under %s", ErrUEFIVarStoreUnusable, ref.Key, uefistore.KeyPrefix(instance.ID))
-	}
-	if m.deps.InstanceTypes == nil {
-		return errors.New("InstanceTypeResolver not wired")
-	}
-	spec, ok := m.deps.InstanceTypes.Resolve(instance.InstanceType)
-	if !ok {
-		return fmt.Errorf("instance type %s not found", instance.InstanceType)
-	}
-	fw, err := NodeFirmwareFor(spec.Architecture)
-	if err != nil {
-		return err
-	}
-	if err := m.checkUEFIFirmware(instance, ref, fw); err != nil {
-		return err
-	}
-
-	dir, err := uefiInstanceDir(deps.DataDir, instance.ID)
-	if err != nil {
-		return err
-	}
-	if reuseUEFIVars(dir, ref.Key, fw.Identity.VarsSize) {
-		recordUEFIStoreSeed("local")
-		slog.InfoContext(ctx, "Reusing this node's UEFI variable store", "instanceId", instance.ID, "key", ref.Key)
-		return nil
-	}
-
-	start := time.Now()
-	region, err := fetchUEFIVars(ctx, deps, ref.Key, fw.Identity.VarsSize)
-	if err != nil {
-		return err
-	}
-	if err := writeFileAtomic(dir, uefiVarsFileName, region); err != nil {
-		return fmt.Errorf("write UEFI variable store for %s: %w", instance.ID, err)
-	}
-	seed, err := json.Marshal(uefiSeed{Key: ref.Key, SeededAt: time.Now().UTC()})
-	if err != nil {
-		return fmt.Errorf("encode %s for %s: %w", uefiSeededFileName, instance.ID, err)
-	}
-	if err := writeFileAtomic(dir, uefiSeededFileName, seed); err != nil {
-		return fmt.Errorf("write %s for %s: %w", uefiSeededFileName, instance.ID, err)
-	}
-	recordUEFIStoreSeed("object")
-	slog.InfoContext(ctx, "Seeded UEFI variable store from the object store",
-		"instanceId", instance.ID, "key", ref.Key, "bytes", len(region), "fetch_ms", otelsetup.Millis(time.Since(start)))
-	return nil
-}
-
-// checkUEFIFirmware fails a store whose arch or VARS size differs from this
-// node's, naming both firmwares in the log and the StateReason. A different
-// CODE build boots, because a package update changes it on every node.
-func (m *Manager) checkUEFIFirmware(instance *VM, ref *UEFIVarStoreRef, fw NodeFirmware) error {
-	stored, node := ref.Firmware, fw.Identity
-	var kind string
-	switch {
-	case stored.Arch != node.Arch:
-		kind = "arch"
-	case stored.VarsSize != node.VarsSize:
-		kind = "vars_size"
-	case stored.CodeSHA256 != node.CodeSHA256:
-		recordUEFIFirmwareMismatch("code")
-		slog.Warn("UEFI variable store was written under a different CODE build; booting it",
-			"instanceId", instance.ID, "key", ref.Key, "writerNode", ref.WriterNode,
-			"storeCodeSha256", stored.CodeSHA256, "nodeCodeSha256", node.CodeSHA256, "nodeCodePath", fw.Code)
-		return nil
-	default:
-		return nil
-	}
-
-	recordUEFIFirmwareMismatch(kind)
-	msg := fmt.Sprintf("%s: the UEFI variable store %s was written under firmware arch=%s vars_size=%d code_sha256=%s, and this node runs arch=%s vars_size=%d code_sha256=%s (%s)",
-		uefiStateReasonCode, ref.Key, stored.Arch, stored.VarsSize, stored.CodeSHA256,
-		node.Arch, node.VarsSize, node.CodeSHA256, fw.Code)
-	m.Inspect(instance, func(v *VM) {
-		if v.Instance != nil {
-			v.Instance.StateReason = &ec2.StateReason{Code: aws.String(uefiStateReasonCode), Message: aws.String(msg)}
-		}
-	})
-	slog.Error("UEFI variable store firmware does not match this node; refusing the launch",
-		"instanceId", instance.ID, "key", ref.Key, "writerNode", ref.WriterNode, "mismatch", kind,
-		"storeArch", stored.Arch, "storeVarsSize", stored.VarsSize, "storeCodeSha256", stored.CodeSHA256,
-		"nodeArch", node.Arch, "nodeVarsSize", node.VarsSize, "nodeCodeSha256", node.CodeSHA256, "nodeCodePath", fw.Code)
-	return fmt.Errorf("%w: %s", ErrUEFIFirmwareMismatch, msg)
-}
-
-// reuseUEFIVars reports whether dir already holds the store key names: a
-// seeded.json naming key beside a vars.fd of the region's size.
-func reuseUEFIVars(dir, key string, varsSize int64) bool {
-	data, err := os.ReadFile(filepath.Join(dir, uefiSeededFileName))
-	if err != nil {
-		return false
-	}
-	var seed uefiSeed
-	if err := json.Unmarshal(data, &seed); err != nil || seed.Key != key {
-		return false
-	}
-	info, err := os.Stat(filepath.Join(dir, uefiVarsFileName))
-	if err != nil || info.Size() != varsSize {
-		slog.Warn("UEFI variable store seeded here has no usable vars.fd; fetching it again", "dir", dir, "key", key)
-		return false
-	}
-	return true
-}
-
-// fetchUEFIVars gets key and returns its decompressed region, which must be
-// exactly varsSize bytes. A failed get is retryable; a missing or malformed
-// object is not.
-func fetchUEFIVars(ctx context.Context, deps UEFIVarStoreDeps, key string, varsSize int64) ([]byte, error) {
-	getCtx, cancel := context.WithTimeout(ctx, uefiStoreGetTimeout)
-	defer cancel()
-
-	out, err := deps.Objects.GetObject(getCtx, &s3.GetObjectInput{Bucket: aws.String(deps.Bucket), Key: aws.String(key)})
-	if err != nil {
-		if objectstore.IsNoSuchKeyError(err) {
-			return nil, fmt.Errorf("%w: %s does not exist", ErrUEFIVarStoreUnusable, key)
-		}
-		return nil, fmt.Errorf("get UEFI variable store %s: %w: %w", key, err, ErrMountRetryable)
-	}
-	defer out.Body.Close()
-
-	limit := varsSize + uefiStoreBodySlack
-	raw, err := io.ReadAll(io.LimitReader(out.Body, limit+1))
-	if err != nil {
-		return nil, fmt.Errorf("read UEFI variable store %s: %w: %w", key, err, ErrMountRetryable)
-	}
-	if int64(len(raw)) > limit {
-		return nil, fmt.Errorf("%w: %s is over %d bytes compressed", ErrUEFIVarStoreUnusable, key, limit)
-	}
-
-	gz, err := gzip.NewReader(bytes.NewReader(raw))
-	if err != nil {
-		return nil, fmt.Errorf("%w: %s is not gzip: %w", ErrUEFIVarStoreUnusable, key, err)
-	}
-	region, err := io.ReadAll(io.LimitReader(gz, varsSize+1))
-	if err != nil {
-		return nil, fmt.Errorf("%w: %s does not decompress: %w", ErrUEFIVarStoreUnusable, key, err)
-	}
-	if int64(len(region)) != varsSize {
-		return nil, fmt.Errorf("%w: %s holds a %d-byte region, and this node's VARS template is %d bytes",
-			ErrUEFIVarStoreUnusable, key, len(region), varsSize)
-	}
-	return region, nil
-}
-
-// writeFileAtomic replaces dir/name with data by temporary file and rename,
-// syncing both, so a crash leaves the old file or the new one.
-func writeFileAtomic(dir, name string, data []byte) error {
-	if err := os.MkdirAll(dir, 0o700); err != nil {
-		return err
-	}
-	tmp, err := os.CreateTemp(dir, "."+name+".*")
-	if err != nil {
-		return err
-	}
-	tmpName := tmp.Name()
-	if _, err := tmp.Write(data); err != nil {
-		_ = tmp.Close()
-		_ = os.Remove(tmpName)
-		return err
-	}
-	if err := tmp.Sync(); err != nil {
-		_ = tmp.Close()
-		_ = os.Remove(tmpName)
-		return err
-	}
-	if err := tmp.Close(); err != nil {
-		_ = os.Remove(tmpName)
-		return err
-	}
-	if err := os.Rename(tmpName, filepath.Join(dir, name)); err != nil {
-		_ = os.Remove(tmpName)
-		return err
-	}
-	d, err := os.Open(dir)
-	if err != nil {
-		return err
-	}
-	defer d.Close()
-	return d.Sync()
-}
-
-// uefiStoreMeterName is the instrumentation scope for variable store launches.
-const uefiStoreMeterName = "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/uefistore"
-
-var (
-	uefiStoreMetricsOnce   sync.Once
-	uefiStoreSeedCounter   metric.Int64Counter
-	uefiFirmwareMismatches metric.Int64Counter
-)
-
-func uefiStoreMetrics() {
-	uefiStoreMetricsOnce.Do(func() {
-		meter := otel.Meter(uefiStoreMeterName)
-		var err error
-		uefiStoreSeedCounter, err = meter.Int64Counter("mulga.vm.uefi_store.seeds",
-			metric.WithDescription("UEFI launches by where their variable store came from: this node's copy or the object store."),
-			metric.WithUnit("{launch}"))
-		if err != nil {
-			otel.Handle(err)
-		}
-		uefiFirmwareMismatches, err = meter.Int64Counter("mulga.vm.uefi_store.firmware_mismatches",
-			metric.WithDescription("UEFI launches whose variable store was written under different firmware, by what differed. Only kind=code boots."),
-			metric.WithUnit("{launch}"))
-		if err != nil {
-			otel.Handle(err)
-		}
-	})
-}
-
-func recordUEFIStoreSeed(source string) {
-	uefiStoreMetrics()
-	if uefiStoreSeedCounter != nil {
-		uefiStoreSeedCounter.Add(context.Background(), 1, metric.WithAttributes(attribute.String("source", source)))
-	}
-}
-
-func recordUEFIFirmwareMismatch(kind string) {
-	uefiStoreMetrics()
-	if uefiFirmwareMismatches != nil {
-		uefiFirmwareMismatches.Add(context.Background(), 1, metric.WithAttributes(attribute.String("kind", kind)))
-	}
-}

spinifex/runtime/compute/vm/uefi_varstore_test.go

@@ -1,539 +0,0 @@
-package vm
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/json"
-	"errors"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/attribute"
-	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
-	"go.opentelemetry.io/otel/sdk/metric/metricdata"
-)
-
-const (
-	testUEFIBucket   = "instances"
-	testUEFIVarsSize = 4096
-	testUEFIInstance = "i-0123456789abcdef0"
-)
-
-var (
-	uefiMetricsReaderOnce sync.Once
-	uefiMetricsReader     *sdkmetric.ManualReader
-)
-
-// uefiCounter returns the sum of the named counter's points whose attribute
-// key has value.
-func uefiCounter(t *testing.T, name, key, value string) int64 {
-	t.Helper()
-	uefiMetricsReaderOnce.Do(func() {
-		uefiMetricsReader = sdkmetric.NewManualReader()
-		otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(uefiMetricsReader)))
-	})
-	var rm metricdata.ResourceMetrics
-	require.NoError(t, uefiMetricsReader.Collect(context.Background(), &rm))
-	var total int64
-	for _, sm := range rm.ScopeMetrics {
-		for _, m := range sm.Metrics {
-			sum, ok := m.Data.(metricdata.Sum[int64])
-			if m.Name != name || !ok {
-				continue
-			}
-			for _, dp := range sum.DataPoints {
-				if v, ok := dp.Attributes.Value(attribute.Key(key)); ok && v.AsString() == value {
-					total += dp.Value
-				}
-			}
-		}
-	}
-	return total
-}
-
-// uefiFixture is a node with one x86_64 firmware candidate, an in-memory
-// object store and a data directory.
-type uefiFixture struct {
-	m        *Manager
-	mounter  *fakeVolumeMounter
-	objects  *objectstore.MemoryObjectStore
-	dataDir  string
-	codePath string
-	identity uefistore.FirmwareIdentity
-}
-
-func newUEFIFixture(t *testing.T) *uefiFixture {
-	t.Helper()
-	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
-	fwDir := t.TempDir()
-	code := filepath.Join(fwDir, "OVMF_CODE.fd")
-	vars := filepath.Join(fwDir, "OVMF_VARS.fd")
-	codeBytes := []byte("ovmf code build 1")
-	require.NoError(t, os.WriteFile(code, codeBytes, 0o600))
-	require.NoError(t, os.WriteFile(vars, make([]byte, testUEFIVarsSize), 0o600))
-	orig := FirmwarePathCandidates
-	FirmwarePathCandidates = map[string][]FirmwareCandidate{"x86_64": {{Code: code, VarsTemplate: vars}}}
-	t.Cleanup(func() { FirmwarePathCandidates = orig })
-
-	sum := sha256.Sum256(codeBytes)
-	f := &uefiFixture{
-		mounter:  &fakeVolumeMounter{},
-		objects:  objectstore.NewMemoryObjectStore(),
-		dataDir:  t.TempDir(),
-		codePath: code,
-		identity: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: testUEFIVarsSize, CodeSHA256: hex.EncodeToString(sum[:])},
-	}
-	f.m = NewManagerWithDeps(Deps{
-		VolumeMounter: f.mounter,
-		InstanceTypes: fakeInstanceTypeResolver{"t3.nano": {Architecture: "x86_64", VCPUs: 1, MemoryMiB: 512}},
-		UEFIVarStore:  UEFIVarStoreDeps{Objects: f.objects, Bucket: testUEFIBucket, DataDir: f.dataDir},
-	})
-	return f
-}
-
-// region is a VARS region of the fixture's size whose bytes say which store it is.
-func region(tag string) []byte {
-	b := make([]byte, testUEFIVarsSize)
-	copy(b, tag)
-	return b
-}
-
-func gzipped(t *testing.T, data []byte) []byte {
-	t.Helper()
-	var buf bytes.Buffer
-	zw := gzip.NewWriter(&buf)
-	_, err := zw.Write(data)
-	require.NoError(t, err)
-	require.NoError(t, zw.Close())
-	return buf.Bytes()
-}
-
-func (f *uefiFixture) putObject(t *testing.T, key string, body []byte) {
-	t.Helper()
-	_, err := f.objects.PutObject(context.Background(), &s3.PutObjectInput{
-		Bucket: aws.String(testUEFIBucket), Key: aws.String(key), Body: bytes.NewReader(body),
-	})
-	require.NoError(t, err)
-}
-
-func (f *uefiFixture) objectBody(t *testing.T, key string) []byte {
-	t.Helper()
-	out, err := f.objects.GetObject(context.Background(), &s3.GetObjectInput{Bucket: aws.String(testUEFIBucket), Key: aws.String(key)})
-	require.NoError(t, err)
-	defer out.Body.Close()
-	b, err := io.ReadAll(out.Body)
-	require.NoError(t, err)
-	return b
-}
-
-func (f *uefiFixture) instanceDir() string {
-	return filepath.Join(f.dataDir, "uefi", testUEFIInstance)
-}
-
-func (f *uefiFixture) writeLocal(t *testing.T, seededKey string, vars []byte) {
-	t.Helper()
-	require.NoError(t, os.MkdirAll(f.instanceDir(), 0o700))
-	require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "vars.fd"), vars, 0o600))
-	seed, err := json.Marshal(uefiSeed{Key: seededKey, SeededAt: time.Now()})
-	require.NoError(t, err)
-	require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "seeded.json"), seed, 0o600))
-}
-
-func (f *uefiFixture) localVars(t *testing.T) []byte {
-	t.Helper()
-	b, err := os.ReadFile(filepath.Join(f.instanceDir(), "vars.fd"))
-	require.NoError(t, err)
-	return b
-}
-
-func (f *uefiFixture) seededKey(t *testing.T) string {
-	t.Helper()
-	b, err := os.ReadFile(filepath.Join(f.instanceDir(), "seeded.json"))
-	require.NoError(t, err)
-	var seed uefiSeed
-	require.NoError(t, json.Unmarshal(b, &seed))
-	return seed.Key
-}
-
-func (f *uefiFixture) instance(key string, fw uefistore.FirmwareIdentity) *VM {
-	v := &VM{
-		ID: testUEFIInstance, InstanceType: "t3.nano", BootMode: "uefi", Status: StateStopped,
-		Instance: &ec2.Instance{},
-	}
-	if key != "" {
-		v.UEFIVarStore = &UEFIVarStoreRef{Key: key, Firmware: fw, WriterNode: "node-a", WrittenAt: time.Unix(1700000000, 0).UTC()}
-	}
-	f.m.Insert(v)
-	return v
-}
-
-func keyFor(name string) string { return uefistore.KeyPrefix(testUEFIInstance) + name }
-
-func TestPrepareUEFIVarStore_FetchesTheObjectAndRecordsTheSeed(t *testing.T) {
-	f := newUEFIFixture(t)
-	key := keyFor("11111111-1111-1111-1111-111111111111")
-	f.putObject(t, key, gzipped(t, region("converted")))
-	instance := f.instance(key, f.identity)
-	before := uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "object")
-
-	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
-
-	assert.Equal(t, region("converted"), f.localVars(t), "vars.fd must hold the decompressed region")
-	assert.Equal(t, key, f.seededKey(t))
-	assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "object"))
-	assert.Equal(t, key, instance.UEFIVarStore.Key, "seeding only reads the pointer")
-}
-
-// A start on the node that seeded the pointer boots that node's copy, which
-// can be newer than the object, and makes no get at all.
-func TestPrepareUEFIVarStore_ReusesTheLocalCopyWhenSeededNamesThePointer(t *testing.T) {
-	f := newUEFIFixture(t)
-	key := keyFor("22222222-2222-2222-2222-222222222222")
-	f.writeLocal(t, key, region("guest wrote this"))
-	instance := f.instance(key, f.identity)
-	before := uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "local")
-
-	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance), "no object exists, so a get would fail")
-
-	assert.Equal(t, region("guest wrote this"), f.localVars(t))
-	assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.seeds", "source", "local"))
-}
-
-// Once the pointer has moved past what this node seeded, the local copy is
-// stale and the pointed object replaces it.
-func TestPrepareUEFIVarStore_RefetchesWhenTheLocalCopyIsNotThePointer(t *testing.T) {
-	tests := []struct {
-		name  string
-		local func(f *uefiFixture, t *testing.T)
-	}{
-		{name: "seeded from an older pointer", local: func(f *uefiFixture, t *testing.T) {
-			f.writeLocal(t, keyFor("older"), region("stale"))
-		}},
-		{name: "vars.fd of the wrong size", local: func(f *uefiFixture, t *testing.T) {
-			f.writeLocal(t, keyFor("current"), []byte("short"))
-		}},
-		{name: "seeded.json unreadable", local: func(f *uefiFixture, t *testing.T) {
-			f.writeLocal(t, keyFor("current"), region("stale"))
-			require.NoError(t, os.WriteFile(filepath.Join(f.instanceDir(), "seeded.json"), []byte("{"), 0o600))
-		}},
-		{name: "no vars.fd", local: func(f *uefiFixture, t *testing.T) {
-			f.writeLocal(t, keyFor("current"), region("stale"))
-			require.NoError(t, os.Remove(filepath.Join(f.instanceDir(), "vars.fd")))
-		}},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			f := newUEFIFixture(t)
-			key := keyFor("current")
-			tt.local(f, t)
-			f.putObject(t, key, gzipped(t, region("restored")))
-			instance := f.instance(key, f.identity)
-
-			require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
-
-			assert.Equal(t, region("restored"), f.localVars(t))
-			assert.Equal(t, key, f.seededKey(t))
-		})
-	}
-}
-
-// A store pflash cannot load fails the launch before any volume is mounted,
-// names both firmwares, and leaves the object, the pointer and the local
-// copy as they were.
-func TestRun_UEFIFirmwareMismatchFailsTheLaunchAndChangesNothing(t *testing.T) {
-	tests := []struct {
-		name   string
-		kind   string
-		mutate func(*uefistore.FirmwareIdentity)
-		want   []string
-	}{
-		{name: "arch", kind: "arch", mutate: func(fw *uefistore.FirmwareIdentity) { fw.Arch = "arm64" },
-			want: []string{"arch=arm64", "arch=x86_64"}},
-		{name: "vars size", kind: "vars_size", mutate: func(fw *uefistore.FirmwareIdentity) { fw.VarsSize = 67108864 },
-			want: []string{"vars_size=67108864", "vars_size=4096"}},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			f := newUEFIFixture(t)
-			key := keyFor("mismatch")
-			body := gzipped(t, region("other firmware"))
-			f.putObject(t, key, body)
-			f.writeLocal(t, keyFor("older"), region("local"))
-			stored := f.identity
-			tt.mutate(&stored)
-			instance := f.instance(key, stored)
-			wantRef := *instance.UEFIVarStore
-			before := uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", tt.kind)
-
-			err := f.m.Run(t.Context(), instance)
-
-			require.ErrorIs(t, err, ErrUEFIFirmwareMismatch)
-			require.NotErrorIs(t, err, ErrMountRetryable, "a mismatch does not clear with time")
-			assert.Empty(t, f.mounter.mounted, "nothing is mounted for a store that cannot boot")
-			require.NotNil(t, instance.Instance.StateReason)
-			assert.Equal(t, "Server.InternalError", aws.StringValue(instance.Instance.StateReason.Code))
-			for _, s := range append(tt.want, f.identity.CodeSHA256, f.codePath, key) {
-				assert.Contains(t, aws.StringValue(instance.Instance.StateReason.Message), s)
-			}
-			assert.Equal(t, wantRef, *instance.UEFIVarStore, "the pointer is unchanged")
-			assert.Equal(t, body, f.objectBody(t, key), "the object is unchanged")
-			assert.Equal(t, region("local"), f.localVars(t), "the local copy is unchanged")
-			assert.Equal(t, keyFor("older"), f.seededKey(t))
-			assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", tt.kind))
-		})
-	}
-}
-
-// An ovmf package update changes the CODE hash on every node, so a store
-// written under another build boots and the difference is counted.
-func TestPrepareUEFIVarStore_DifferentCodeBuildBootsAndIsCounted(t *testing.T) {
-	f := newUEFIFixture(t)
-	key := keyFor("code")
-	f.putObject(t, key, gzipped(t, region("older ovmf")))
-	stored := f.identity
-	stored.CodeSHA256 = strings.Repeat("0", 64)
-	instance := f.instance(key, stored)
-	before := uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", "code")
-
-	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), instance))
-
-	assert.Equal(t, region("older ovmf"), f.localVars(t))
-	assert.Nil(t, instance.Instance.StateReason)
-	assert.Equal(t, before+1, uefiCounter(t, "mulga.vm.uefi_store.firmware_mismatches", "kind", "code"))
-}
-
-// failingGetStore fails every get, standing in for an object store that
-// does not answer.
-type failingGetStore struct {
-	*objectstore.MemoryObjectStore
-
-	err error
-}
-
-func (s *failingGetStore) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
-	return nil, s.err
-}
-
-func TestPrepareUEFIVarStore_FailedGetIsRetryable(t *testing.T) {
-	f := newUEFIFixture(t)
-	f.m.deps.UEFIVarStore.Objects = &failingGetStore{MemoryObjectStore: f.objects, err: errors.New("connection refused")}
-	instance := f.instance(keyFor("down"), f.identity)
-
-	err := f.m.prepareUEFIVarStore(t.Context(), instance)
-
-	require.ErrorIs(t, err, ErrMountRetryable, "restore backs off on a get as it does on a mount")
-	assert.NoFileExists(t, filepath.Join(f.instanceDir(), "seeded.json"))
-}
-
-// An object that is missing or does not decode to this node's region is
-// refused for good, and nothing is written locally.
-func TestPrepareUEFIVarStore_UnusableObjectIsRefused(t *testing.T) {
-	tests := []struct {
-		name string
-		body []byte
-	}{
-		{name: "missing"},
-		{name: "not gzip", body: region("raw, uncompressed")},
-		{name: "short region", body: gzipped(t, make([]byte, testUEFIVarsSize-1))},
-		{name: "long region", body: gzipped(t, make([]byte, testUEFIVarsSize+1))},
-		{name: "truncated gzip", body: gzipped(t, region("cut"))[:20]},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			f := newUEFIFixture(t)
-			key := keyFor("bad")
-			if tt.body != nil {
-				f.putObject(t, key, tt.body)
-			}
-			instance := f.instance(key, f.identity)
-
-			err := f.m.prepareUEFIVarStore(t.Context(), instance)
-
-			require.ErrorIs(t, err, ErrUEFIVarStoreUnusable)
-			require.NotErrorIs(t, err, ErrMountRetryable)
-			assert.NoFileExists(t, filepath.Join(f.instanceDir(), "vars.fd"))
-			assert.NoFileExists(t, filepath.Join(f.instanceDir(), "seeded.json"))
-		})
-	}
-}
-
-func TestPrepareUEFIVarStore_PointerOutsideTheInstancePrefixIsRefused(t *testing.T) {
-	for _, key := range []string{"uefi/i-0aaaaaaaaaaaaaaaa/other", keyFor("a/b"), uefistore.KeyPrefix(testUEFIInstance)} {
-		t.Run(key, func(t *testing.T) {
-			f := newUEFIFixture(t)
-			f.putObject(t, key, gzipped(t, region("x")))
-			instance := f.instance(key, f.identity)
-
-			require.ErrorIs(t, f.m.prepareUEFIVarStore(t.Context(), instance), ErrUEFIVarStoreUnusable)
-		})
-	}
-}
-
-// A BIOS instance, a microvm and a legacy instance with no pointer touch
-// neither the object store nor the data directory.
-func TestPrepareUEFIVarStore_NoPointerIsANoOp(t *testing.T) {
-	f := newUEFIFixture(t)
-	f.m.deps.UEFIVarStore.Objects = &failingGetStore{MemoryObjectStore: f.objects, err: errors.New("must not be called")}
-	bios := f.instance(keyFor("bios"), f.identity)
-	bios.BootMode = ""
-	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), bios))
-
-	legacy := f.instance("", f.identity)
-	require.NoError(t, f.m.prepareUEFIVarStore(t.Context(), legacy))
-	assert.NoDirExists(t, filepath.Join(f.dataDir, "uefi"))
-}
-
-// A UEFI instance with no pointer, or two stores, is refused before the
-// launch clears or changes anything on its record.
-func TestRun_UEFIInstanceWithoutExactlyOneStoreIsRefused(t *testing.T) {
-	efiVolume := EBSRequest{Name: "vol-1-efi", EFI: true}
-	tests := []struct {
-		name    string
-		pointer bool
-		efi     bool
-		want    error
-	}{
-		{name: "no pointer and no EFI volume", want: ErrUEFIVarStoreMissing},
-		{name: "a pointer and an EFI volume", pointer: true, efi: true, want: ErrUEFIVarStoreConflict},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			f := newUEFIFixture(t)
-			key := ""
-			if tt.pointer {
-				key = keyFor("p")
-			}
-			instance := f.instance(key, f.identity)
-			if tt.efi {
-				instance.EBSRequests.Requests = []EBSRequest{efiVolume}
-			}
-			stale := &ec2.StateReason{Code: aws.String("Client.UserInitiatedShutdown"), Message: aws.String("Client.UserInitiatedShutdown: User initiated shutdown")}
-			instance.Instance.StateReason = stale
-			before := instance.Record()
-
-			err := f.m.Run(t.Context(), instance)
-
-			require.ErrorIs(t, err, tt.want)
-			assert.Empty(t, f.mounter.mounted)
-			assert.Equal(t, before, instance.Record(), "the record is unchanged")
-		})
-	}
-}
-
-// Start from StateError transitions to pending before launching; a refused
-// UEFI instance must not take that transition either.
-func TestStart_UEFIInstanceWithoutAPointerIsRefusedBeforeTheTransition(t *testing.T) {
-	f := newUEFIFixture(t)
-	transitions := 0
-	f.m.deps.TransitionState = func(v *VM, s InstanceState) error { transitions++; v.Status = s; return nil }
-	instance := f.instance("", f.identity)
-	instance.Status = StateError
-
-	err := f.m.Start(t.Context(), instance.ID)
-
-	require.ErrorIs(t, err, ErrUEFIVarStoreMissing)
-	assert.Equal(t, 0, transitions)
-	assert.Equal(t, StateError, instance.Status)
-}
-
-// Until RunInstances seeds pointers, an instance whose variables are on an
-// EFI volume still launches through the volume, and a BIOS or microvm
-// guest needs no store at all.
-func TestCheckUEFIVarStore_Accepts(t *testing.T) {
-	tests := []struct {
-		name string
-		v    *VM
-	}{
-		{name: "EFI volume", v: &VM{BootMode: "uefi", EBSRequests: EBSRequests{Requests: []EBSRequest{{Name: "vol-efi", EFI: true}}}}},
-		{name: "pointer", v: &VM{BootMode: "uefi-preferred", UEFIVarStore: &UEFIVarStoreRef{Key: "k"}}},
-		{name: "BIOS", v: &VM{BootMode: "legacy-bios"}},
-		{name: "no boot mode", v: &VM{}},
-		{name: "microvm", v: &VM{BootMode: "uefi", DirectBoot: true}},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			assert.NoError(t, CheckUEFIVarStore(tt.v))
-		})
-	}
-}
-
-// A launch from a pointer opens this node's copy as pflash unit 1 with
-// format=raw, after the read-only CODE blob.
-func TestStartQEMU_PointerPassesTheLocalCopyAsPflashUnit1(t *testing.T) {
-	f := newUEFIFixture(t)
-	f.m.deps.NetworkPlumber = &fakeNetworkPlumber{}
-	launches := withFakeQEMU(t)
-	key := keyFor("boot")
-	f.writeLocal(t, key, region("seeded"))
-	instance := f.instance(key, f.identity)
-	instance.EBSRequests.Requests = []EBSRequest{{Name: "vol-root", NBDURI: nbdListener(t), Boot: true}}
-
-	require.NoError(t, f.m.startQEMU(instance, false))
-
-	got := launches()
-	require.Len(t, got, 1)
-	vars := filepath.Join(f.instanceDir(), "vars.fd")
-	assert.Equal(t, []string{
-		"file=" + f.codePath + ",format=raw,if=pflash,unit=0,readonly=on",
-		"file=" + vars + ",format=raw,if=pflash,unit=1",
-	}, allArgValues(got[0], "-drive"))
-}
-
-// A reboot reuses the copy the launch seeded; without one there is nothing
-// safe to boot, so QEMU is not started.
-func TestStartQEMU_PointerWithoutALocalCopyIsNotLaunched(t *testing.T) {
-	f := newUEFIFixture(t)
-	f.m.deps.NetworkPlumber = &fakeNetworkPlumber{}
-	launches := withFakeQEMU(t)
-	instance := f.instance(keyFor("boot"), f.identity)
-	instance.EBSRequests.Requests = []EBSRequest{{Name: "vol-root", NBDURI: nbdListener(t), Boot: true}}
-
-	err := f.m.startQEMU(instance, false)
-
-	require.ErrorIs(t, err, os.ErrNotExist)
-	assert.Empty(t, launches())
-}
-
-func TestUEFIInstanceDir_RefusesIDsThatAreNotOnePathSegment(t *testing.T) {
-	for _, id := range []string{"", "..", "i-../x", "i-a/b", `i-a\b`, "vol-1"} {
-		_, err := uefiInstanceDir("/data", id)
-		assert.Error(t, err, id)
-	}
-	dir, err := uefiInstanceDir("/data", testUEFIInstance)
-	require.NoError(t, err)
-	assert.Equal(t, "/data/uefi/"+testUEFIInstance, dir)
-}
-
-// The CODE hash is what a store records; a package update that replaces the
-// blob must be hashed again rather than served from the cache.
-func TestNodeFirmwareFor_HashesTheCodeBlobAgainAfterItChanges(t *testing.T) {
-	f := newUEFIFixture(t)
-	first, err := NodeFirmwareFor("x86_64")
-	require.NoError(t, err)
-	assert.Equal(t, f.identity, first.Identity)
-
-	updated := []byte("ovmf code build 2, a longer blob")
-	require.NoError(t, os.WriteFile(f.codePath, updated, 0o600))
-	second, err := NodeFirmwareFor("x86_64")
-	require.NoError(t, err)
-
-	sum := sha256.Sum256(updated)
-	assert.Equal(t, hex.EncodeToString(sum[:]), second.Identity.CodeSHA256)
-	assert.Equal(t, f.codePath, second.Code)
-}

spinifex/runtime/compute/vm/vm.go

@@ -387,7 +387,7 @@ type Device struct {
 }
 
 // Drive is one QEMU -drive entry, used for the pflash firmware drives (the
-// CODE blob and the variable store). Each set field becomes a key=value option.
+// CODE blob and the EFI volume). Each set field becomes a key=value option.
 type Drive struct {
 	File   string `json:"file"`
 	Format string `json:"format"`
@@ -396,7 +396,7 @@ type Drive struct {
 	ID     string `json:"id"`
 	Cache  string `json:"cache,omitempty"`
 	// Unit selects the pflash slot when If=="pflash": 0 is the CODE blob,
-	// 1 is the per-VM variable store. Ignored for non-pflash drives.
+	// 1 is the per-VM VARS volume. Ignored for non-pflash drives.
 	Unit int `json:"unit,omitempty"`
 	// ReadOnly emits readonly=on; used for the pflash CODE blob.
 	ReadOnly bool `json:"readonly,omitempty"`
@@ -588,8 +588,8 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 			return nil, fwErr
 		}
 		// pflash CODE must come before VARS so QEMU loads firmware into the
-		// readonly slot first; startQEMU emits the VARS drive (unit 1) from
-		// the node's copy of the variable store or the EFI volume.
+		// readonly slot first; buildDrives emits the VARS drive (unit 1) from
+		// the per-VM EFI viperblock volume.
 		drives = append([]Drive{{If: "pflash", Format: "raw", Unit: 0, ReadOnly: true, File: codePath}}, drives...)
 	}
 

spinifex/runtime/compute/vm/volume_mounter.go

@@ -5,8 +5,8 @@ import (
 )
 
 // VolumeMounter mounts and unmounts the EBS volumes attached to a VM. The
-// real implementation routes ebs.mount / ebs.unmount NATS requests; the
-// abstraction keeps NATS out of the manager.
+// real implementation publishes and unpublishes them through the EBS provider
+// over NATS; the abstraction keeps NATS out of the manager.
 //
 // Every method takes the caller's context so the ebs.* span joins the trace
 // that asked for the work rather than rooting one of its own. Shutdown and
@@ -14,9 +14,9 @@ import (
 // honest answer rather than a missing one.
 type VolumeMounter interface {
 	// Mount mounts every attached volume in v.EBSRequests.Requests, recording
-	// the resolved NBDURI back onto each request entry.
+	// the resolved export back onto each request entry.
 	Mount(ctx context.Context, v *VM) error
-	// Unmount sends ebs.unmount for each attached volume, which drives the
+	// Unmount unmounts each attached volume, which drives the
 	// synchronous block-map seal. Every volume is attempted; the seal
 	// failures are aggregated and returned, and mean the data is not durable.
 	Unmount(ctx context.Context, v *VM) error
@@ -31,12 +31,12 @@ type VolumeMounter interface {
 	// anywhere.
 	Abandon(ctx context.Context, v *VM, reason string) error
 
-	// MountOne sends ebs.mount for a single request and writes the resolved
-	// NBDURI back into req.NBDURI on success. Used by hot-attach. accountID
+	// MountOne mounts a single request and writes the resolved export back
+	// into req on success. Used by hot-attach. accountID
 	// names the owner, which a lone request carries no instance to supply.
 	MountOne(ctx context.Context, accountID string, req *EBSRequest) error
-	// UnmountOne sends ebs.unmount for a single request and returns any error.
-	// ebs.unmount drives the synchronous block-map seal to predastore, so hot
+	// UnmountOne unmounts a single request and returns any error. The
+	// unmount drives the synchronous block-map seal to the backend, so hot
 	// detach gates the volume's available transition on the returned error.
 	UnmountOne(ctx context.Context, accountID string, req EBSRequest) error
 }

spinifex/runtime/compute/vm/volumes.go

@@ -25,7 +25,7 @@ const blockdevDelMaxAttempts = 20
 // object-del steps), so a wedged-but-responsive QEMU cannot stack per-step
 // timeouts into an effectively unbounded hang. It comfortably covers those
 // steps' own budgets (qmpCommandTimeout + the bounded blockdev-del retry) plus
-// margin; the ebs.unmount seal keeps its separate unmountSealTimeout.
+// margin; the volume seal is not bound by it and keeps its own budget.
 const detachAggregateTimeout = 3 * time.Minute
 
 // ErrHotAttachUnsupported is returned by AttachVolume for a direct-boot microvm,
@@ -119,9 +119,9 @@ func (m *Manager) AttachVolume(ctx context.Context, id, volumeID, device string)
 		return "", fmt.Errorf("VolumeMounter not wired")
 	}
 	if err := m.deps.VolumeMounter.MountOne(ctx, instance.AccountID, &ebsRequest); err != nil {
-		slog.ErrorContext(ctx, "AttachVolume: ebs.mount failed", "volumeId", volumeID, "err", err)
-		// Empty-URI response leaves backend NBD state ambiguous; unmount
-		// defensively to avoid orphaning a half-started mount.
+		slog.ErrorContext(ctx, "AttachVolume: publish failed", "volumeId", volumeID, "err", err)
+		// A publish with no usable reply may have started an export, so it is
+		// unpublished defensively rather than orphaned.
 		if errors.Is(err, ErrMountAmbiguous) {
 			_ = m.rollbackUnmount(ctx, instance.AccountID, ebsRequest)
 		}
@@ -300,7 +300,7 @@ func (m *Manager) AttachVolume(ctx context.Context, id, volumeID, device string)
 }
 
 // DetachVolume hot-unplugs a volume via the QMP pipeline (device_del →
-// blockdev-del → object-del → ebs.unmount). Returns the recorded AWS-API device
+// blockdev-del → object-del → unpublish). Returns the recorded AWS-API device
 // name. force=true tolerates device_del failure; blockdev-del is always required.
 func (m *Manager) DetachVolume(ctx context.Context, id, volumeID, device string, force bool) (string, error) {
 	instance, ok := m.Get(id)
@@ -344,8 +344,8 @@ func (m *Manager) DetachVolume(ctx context.Context, id, volumeID, device string,
 
 	// Bound the ctx-driven portion of the hot-unplug chain so a wedged-but-
 	// responsive QEMU cannot stack per-step QMP timeouts into an unbounded hang
-	// that pins attachMu forever. The ebs.unmount seal below keeps its own
-	// unmountSealTimeout. This deadline cannot preempt a plain-mutex QMP call,
+	// that pins attachMu forever. The volume seal below is not bound by it and
+	// keeps its own budget. This deadline cannot preempt a plain-mutex QMP call,
 	// which is why a fully dead QEMU is short-circuited outright below.
 	ctx, cancel := context.WithTimeout(ctx, detachAggregateTimeout)
 	defer cancel()
@@ -359,7 +359,7 @@ func (m *Manager) DetachVolume(ctx context.Context, id, volumeID, device string,
 
 	// A confirmed-dead QEMU took every block node and guest device down with it,
 	// so the QMP unplug steps are moot — and worse, issuing them only risks
-	// hanging on the wedged process. Skip straight to the ebs.unmount seal and
+	// hanging on the wedged process. Skip straight to the unpublish seal and
 	// the attachment clear. This is the detach-wedges-when-QEMU-dies fix: the
 	// context deadline above cannot interrupt a plain-mutex QMP path, so a
 	// provably-gone process must bypass it rather than wait on it. A missing PID
@@ -422,15 +422,15 @@ func (m *Manager) DetachVolume(ctx context.Context, id, volumeID, device string,
 			"volumeId", volumeID, "instanceId", instance.ID)
 	}
 
-	// ebs.unmount drives the synchronous block-map seal to predastore. On
+	// Unpublish drives the synchronous block-map seal to the backend. On
 	// failure the volume's local WAL is retained, so keep the volume attached
 	// and return the error; an AWS-CLI retry re-drives the seal and same-node
 	// reattach still works meanwhile.
 	if m.deps.VolumeMounter != nil {
 		if err := m.deps.VolumeMounter.UnmountOne(ctx, instance.AccountID, ebsReq); err != nil {
-			slog.ErrorContext(ctx, "DetachVolume: ebs.unmount seal failed, leaving volume attached",
+			slog.ErrorContext(ctx, "DetachVolume: unpublish seal failed, leaving volume attached",
 				"volumeId", volumeID, "err", err)
-			return "", fmt.Errorf("ebs.unmount seal: %w", err)
+			return "", fmt.Errorf("unpublish seal: %w", err)
 		}
 	}
 
@@ -554,7 +554,7 @@ func (m *Manager) tryBlockdevDel(ctx context.Context, instance *VM, nodeName str
 			return nil
 		}
 		// A prior detach already removed the block node (e.g. an AWS-CLI retry
-		// after the ebs.unmount seal failed): treat as success so the retry
+		// after the unpublish seal failed): treat as success so the retry
 		// resumes through object-del to the seal instead of wedging on a
 		// now-absent node.
 		if isQMPNodeNotFound(err) {

spinifex/runtime/compute/vm/volumes_test.go

@@ -543,7 +543,7 @@ func TestDetachVolume_DeviceGuards(t *testing.T) {
 }
 
 // TestDetachVolume_SealFailureGatesAvailable covers the durability gate in the
-// detach-seal fix: when ebs.unmount (the synchronous block-map seal) fails,
+// detach-seal fix: when unpublish (the synchronous block-map seal) fails,
 // DetachVolume must propagate the error, must NOT flip the volume to "available",
 // and must leave it in EBSRequests so the volume stays attached/retryable and its
 // local WAL is retained. A regression back to fire-and-forget unmount would let a
@@ -661,7 +661,7 @@ func TestAttachVolume_MountAmbiguous_TriggersUnmountOne(t *testing.T) {
 // MountOne failures (NATS timeout, backend explicit Error, marshal errors)
 // do NOT trigger UnmountOne — only the empty-URI ambiguous case rolls back.
 func TestAttachVolume_GenericMountError_DoesNotUnmount(t *testing.T) {
-	mounter := &fakeVolumeMounter{mountOneErr: errors.New("ebs.mount NATS request: timeout")}
+	mounter := &fakeVolumeMounter{mountOneErr: errors.New("publish request: timeout")}
 	m, _ := attachVolumeRunningInstance(t, nil, mounter)
 
 	_, err := m.AttachVolume(t.Context(), "i-1", "vol-1", "/dev/sdf")
@@ -1387,7 +1387,7 @@ func TestDetachVolume_DeviceDeletedTimeout_FallsBackToRetry(t *testing.T) {
 // detach-wedges-when-QEMU-dies fix: when QEMU is provably gone (PID file
 // present, process dead), DetachVolume issues no QMP command — those steps
 // would only hang on the wedged process — and proceeds straight to the
-// ebs.unmount seal and the attachment clear.
+// unpublish seal and the attachment clear.
 func TestDetachVolume_ConfirmedDeadQEMU_SkipsQMPAndSeals(t *testing.T) {
 	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
 
@@ -1418,7 +1418,7 @@ func TestDetachVolume_ConfirmedDeadQEMU_SkipsQMPAndSeals(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "/dev/sdf", device)
 	assert.False(t, qmpCalled.Load(), "a confirmed-dead QEMU must receive no QMP command")
-	assert.Equal(t, []string{"vol-1"}, mounter.unmountedOne, "the ebs.unmount seal must still run")
+	assert.Equal(t, []string{"vol-1"}, mounter.unmountedOne, "the unpublish seal must still run")
 	calls := updater.snapshot()
 	require.Len(t, calls, 1, "the stale attachment must be cleared exactly once")
 	assert.Equal(t, "available", calls[0].State, "the volume must be returned to available")

spinifex/runtime/daemon/daemon.go

@@ -41,7 +41,6 @@ import (
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
 	telemetryv1 "github.com/mulgadc/spinifex/contracts/telemetry/v1"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/preflight"
 	acmdomain "github.com/mulgadc/spinifex/spinifex/domains/acm"
@@ -152,7 +151,6 @@ type Daemon struct {
 	natsConn          *nats.Conn
 	resourceMgr       *ResourceManager
 	instanceService   *ec2instance.InstanceServiceImpl
-	instanceObjects   objectstore.ObjectStore
 	dnsWriter         *dns.Writer
 	dnsReconciler     *dns.Reconciler
 	dnsBaseDomain     string
@@ -734,6 +732,11 @@ func (d *Daemon) SetConfigPath(path string) {
 
 // NewDaemon creates a new daemon instance.
 func NewDaemon(cfg *config.ClusterConfig) (*Daemon, error) {
+	// The fence notice is node-addressed: a daemon that cannot subscribe to it
+	// would leave a guest running on a volume whose export was revoked.
+	if err := ebsprovider.ValidateNodeID(cfg.Node); err != nil {
+		return nil, fmt.Errorf("cluster config node %q: %w", cfg.Node, err)
+	}
 	ctx, cancel := context.WithCancel(context.Background())
 
 	// If WalDir is not set, use BaseDir
@@ -1354,10 +1357,8 @@ func (d *Daemon) subscribeAll() error {
 		// here, so the reconcile request flows daemon-ward.
 		natsSub{dhcp.TopicLeaseChanged, d.handleDHCPLeaseChanged, "spinifex-workers"},
 		natsSub{dhcp.TopicOwnerCheck, d.handleDHCPOwnerCheck, "spinifex-workers"},
-		// Node-addressed with no queue group: the fenced guest is on this host,
-		// so a worker on another node would find nothing to stop.
-		natsSub{viperblocklegacyv1.VolumeFencedSubject(d.node), d.handleVolumeFenced, ""},
 	)
+	subs = append(subs, d.fenceNoticeSubs()...)
 
 	// Per-node, no queue group: a second daemon claiming this node ID also
 	// answers, so the teardown probe sees a duplicate and refuses to seal. An
@@ -2018,7 +2019,6 @@ func (d *Daemon) startCluster() error {
 
 	// Create services before loading/launching instances, since LaunchInstance depends on them
 	store := objectstore.NewS3ObjectStoreFromConfig(netaddr.DialTarget(d.config.Predastore.Host), d.config.Predastore.Region, d.config.Predastore.AccessKey, d.config.Predastore.SecretKey)
-	d.instanceObjects = store
 	d.instanceService = ec2instance.NewInstanceServiceImpl(d.config, d.resourceMgr.instanceTypes, d.natsConn, store, d.vmMgr, d.resourceMgr, d.jsManager)
 	d.dnsWriter = dns.NewWriter(d.config, d.clusterConfig, d.natsConn)
 	d.dnsReconciler = dns.NewReconciler(d.config, d.clusterConfig, d.natsConn, d.dnsWriter, d.dnsDesiredSet, d.dnsWatchSources()...)
@@ -2624,9 +2624,9 @@ const ebsProviderRequestTimeout = 30 * time.Second
 // a missing responder must be reported promptly, and the check is advisory.
 const ebsProviderProbeTimeout = 2 * time.Second
 
-// configureEBSProvider wires a single NATSProvider into every EBS-adjacent
-// service. The control plane never constructs a provider implementation
-// itself, so this always runs regardless of which daemon answers on the wire.
+// configureEBSProvider wires a NATSProvider into the EBS-adjacent services and
+// the volume mounter. The control plane never constructs a provider
+// implementation, so this runs whichever daemon answers on the wire.
 func (d *Daemon) configureEBSProvider() error {
 	provider := ebsprovider.NewNATSProvider(d.natsConn, ebsProviderRequestTimeout)
 	d.ebsProvider = provider

spinifex/runtime/daemon/daemon_handlers.go

@@ -191,9 +191,6 @@ func handleNATSRequestWithPrincipal[I any, O any](nodeID string, serviceFn func(
 func (d *Daemon) handleEC2Events(msg *nats.Msg) {
 	start := time.Now()
 	action, outcome := d.dispatchEC2Command(msg)
-	if outcome == outcomeDeferred {
-		return
-	}
 	otelsetup.RecordRequest(context.Background(), ec2CmdAction(action), outcome, time.Since(start))
 }
 
@@ -233,21 +230,6 @@ func (d *Daemon) dispatchEC2Command(msg *nats.Msg) (string, string) {
 		return name, d.handleAttachVolume(ctx, msg, command, instance)
 	case command.Attributes.DetachVolume:
 		return name, d.handleDetachVolume(ctx, msg, command, instance)
-	case command.Attributes.DrainVolume:
-		// A drain flushes the volume's whole dirty set to S3, so it is bounded by
-		// the dirty set rather than by a unit of work. nats.go delivers a
-		// subscription's messages serially, so running it inline would stall
-		// every other command for this instance — stop, terminate, hot-plug —
-		// for the duration. It touches no VM state, so it is safe off-thread.
-		//
-		// It therefore records its own point: timing the dispatch would call
-		// every drain an instant success.
-		started := time.Now()
-		go func() {
-			outcome := d.handleDrainVolume(ctx, msg, command, instance)
-			otelsetup.RecordRequest(context.Background(), ec2CmdAction(name), outcome, time.Since(started))
-		}()
-		return name, outcomeDeferred
 	case command.Attributes.AttachENI:
 		return name, d.handleAttachNetworkInterface(ctx, msg, command, instance)
 	case command.Attributes.DetachENI:
@@ -314,8 +296,6 @@ func ec2CommandName(command ec2v1.EC2InstanceCommand) string {
 		return "AttachVolume"
 	case command.Attributes.DetachVolume:
 		return "DetachVolume"
-	case command.Attributes.DrainVolume:
-		return "DrainVolume"
 	case command.Attributes.AttachENI:
 		return "AttachNetworkInterface"
 	case command.Attributes.DetachENI:

spinifex/runtime/daemon/daemon_handlers_fence.go

@@ -3,56 +3,77 @@ package daemon
 import (
 	"context"
 	"encoding/json"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"fmt"
 	"log/slog"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/nats-io/nats.go"
 )
 
-// handleVolumeFenced stops the local guest whose volume viperblockd has just
-// torn the export out from under.
+// fenceNoticeSubs is this node's fence-notice subscription, whatever the EBS
+// provider. Node-addressed with no queue group: the fenced guest is on this
+// host, so a worker on another node would find nothing to stop.
+func (d *Daemon) fenceNoticeSubs() []natsSub {
+	subject, err := ebsprovider.FenceNoticeSubject(d.node)
+	if err != nil {
+		slog.Error("Not serving EBS fence notices: node ID is not a single NATS subject token", "node", d.node, "err", err)
+		return nil
+	}
+	return []natsSub{{subject, d.handleFenceNotice, ""}}
+}
+
+// handleFenceNotice stops the local guest whose volume the EBS provider has
+// just revoked the export of.
 //
 // By the time this arrives the disk is already gone, so the guest is doing no
 // useful work and its writes cannot reach the backend. What is left is a QEMU
 // process holding memory and taps, and an instance record claiming this node
 // runs something that runs somewhere else now.
-func (d *Daemon) handleVolumeFenced(msg *nats.Msg) string {
-	ctx, span := natsmsg.StartConsumerSpan(msg)
+func (d *Daemon) handleFenceNotice(msg *nats.Msg) string {
+	ctx, span := ebsprovider.StartServerSpan(context.Background(), msg)
 	defer span.End()
 
-	var event viperblocklegacyv1.VolumeFencedEvent
-	if err := json.Unmarshal(msg.Data, &event); err != nil {
-		slog.ErrorContext(ctx, "volume fenced: bad event", "err", err)
-		ackVolumeFenced(ctx, msg, viperblocklegacyv1.VolumeFencedAck{Error: "bad event: " + err.Error()})
+	var notice ebsprovider.FenceNotice
+	decodeErr := json.Unmarshal(msg.Data, &notice)
+	ack := ebsprovider.FenceNoticeAck{Versioned: ebsprovider.ReplyVersioned(notice.SchemaVersion)}
+	refuse := func(code ebsprovider.ErrorCode, message string) string {
+		slog.ErrorContext(ctx, "volume fenced: refused the notice", "volumeId", notice.VolumeID, "code", code, "err", message)
+		ack.Error = &ebsprovider.ProviderError{Code: code, Message: message}
+		ebsprovider.RecordProviderError(span, ack.Error)
+		ackFenceNotice(ctx, msg, ack)
 		return outcomeError
 	}
-	if event.Volume == "" {
-		slog.ErrorContext(ctx, "volume fenced: event named no volume")
-		ackVolumeFenced(ctx, msg, viperblocklegacyv1.VolumeFencedAck{Error: "event named no volume"})
-		return outcomeError
+	if decodeErr != nil {
+		return refuse(ebsprovider.ErrorCodeInvalidArgument, "bad notice: "+decodeErr.Error())
+	}
+	if !ebsprovider.SupportsSchemaVersion(notice.SchemaVersion) {
+		return refuse(ebsprovider.ErrorCodeUnsupportedVersion,
+			fmt.Sprintf("unsupported schema version %d; ", notice.SchemaVersion)+ebsprovider.SupportedVersionsText())
+	}
+	if notice.VolumeID == "" {
+		return refuse(ebsprovider.ErrorCodeInvalidArgument, "notice named no volume")
 	}
 
 	// FenceVolume moves the guest out of running before it returns, so the ack
 	// below is sent only once nothing here will resume it.
-	instanceID := d.vmMgr.FenceVolume(ctx, event.Volume, event.Reason)
-	ackVolumeFenced(ctx, msg, viperblocklegacyv1.VolumeFencedAck{Instance: instanceID})
-	if instanceID == "" {
+	ack.InstanceID = d.vmMgr.FenceVolume(ctx, notice.VolumeID, notice.Reason)
+	ackFenceNotice(ctx, msg, ack)
+	if ack.InstanceID == "" {
 		// The usual case: the volume outlived the guest that was using it, so
 		// there is nothing local left to stop.
 		slog.InfoContext(ctx, "volume fenced with no local guest using it",
-			"volumeId", event.Volume, "winner", event.Winner)
+			"volumeId", notice.VolumeID, "winner", notice.Winner)
 		return outcomeSuccess
 	}
 
 	slog.ErrorContext(ctx, "volume fenced, stopped the guest that was using it",
-		"volumeId", event.Volume, "instanceId", instanceID, "winner", event.Winner)
+		"volumeId", notice.VolumeID, "instanceId", ack.InstanceID, "winner", notice.Winner)
 	return outcomeSuccess
 }
 
-// ackVolumeFenced answers viperblockd, which repeats the notice until it hears
+// ackFenceNotice answers the provider, which repeats the notice until it hears
 // back. A notice with no reply subject was published once and needs no answer.
-func ackVolumeFenced(ctx context.Context, msg *nats.Msg, ack viperblocklegacyv1.VolumeFencedAck) {
+func ackFenceNotice(ctx context.Context, msg *nats.Msg, ack ebsprovider.FenceNoticeAck) {
 	if msg.Reply == "" {
 		return
 	}
@@ -62,6 +83,6 @@ func ackVolumeFenced(ctx context.Context, msg *nats.Msg, ack viperblocklegacyv1.
 		return
 	}
 	if err := msg.Respond(payload); err != nil {
-		slog.WarnContext(ctx, "volume fenced: could not acknowledge, viperblockd will repeat the notice", "err", err)
+		slog.WarnContext(ctx, "volume fenced: could not acknowledge, the provider will repeat the notice", "err", err)
 	}
 }

spinifex/runtime/daemon/daemon_handlers_fence_invariants_test.go

@@ -2,92 +2,313 @@ package daemon
 
 import (
 	"encoding/json"
+	"fmt"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/codes"
 )
 
-// fenceDaemon is a daemon running one guest on volumeID, answering fence
-// notices on a real NATS server as it does in production.
+// fenceDaemon is a daemon running one guest on volumeID, subscribed to fence
+// notices on a real NATS server through the subscription it makes in production.
 func fenceDaemon(t *testing.T, volumeID string) (*Daemon, *vm.VM, *nats.Conn) {
+	t.Helper()
+	return fenceDaemonWithManager(t, volumeID, vm.NewManagerWithDeps(vm.Deps{NodeID: "node-a"}))
+}
+
+func fenceDaemonWithManager(t *testing.T, volumeID string, mgr *vm.Manager) (*Daemon, *vm.VM, *nats.Conn) {
 	t.Helper()
 	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
 
 	_, nc := testutil.StartTestNATS(t)
-	d := &Daemon{node: "node-a", vmMgr: vm.NewManagerWithDeps(vm.Deps{NodeID: "node-a"})}
-	guest := &vm.VM{
+	d := &Daemon{
+		node:              "node-a",
+		vmMgr:             mgr,
+		natsConn:          nc,
+		natsSubscriptions: map[string]*nats.Subscription{},
+	}
+	guest := fencedGuest(volumeID)
+	d.vmMgr.Insert(guest)
+
+	require.NoError(t, d.registerNatsSubs(d.fenceNoticeSubs()))
+	require.NoError(t, nc.Flush())
+	return d, guest, nc
+}
+
+func fencedGuest(volumeID string) *vm.VM {
+	return &vm.VM{
 		ID:          "i-fencedguest",
 		Status:      vm.StateRunning,
 		Instance:    &ec2.Instance{},
 		EBSRequests: vm.EBSRequests{Requests: []vm.EBSRequest{{Name: volumeID}}},
 	}
-	d.vmMgr.Insert(guest)
+}
 
-	_, err := nc.Subscribe(viperblocklegacyv1.VolumeFencedSubject(d.node), func(msg *nats.Msg) { d.handleVolumeFenced(msg) })
+// requestFenceNotice sends raw to node-a's fence-notice subject and decodes the ack.
+func requestFenceNotice(t *testing.T, nc *nats.Conn, raw []byte) ebsprovider.FenceNoticeAck {
+	t.Helper()
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
 	require.NoError(t, err)
-	require.NoError(t, nc.Flush())
-	return d, guest, nc
+	reply, err := nc.Request(subject, raw, 2*time.Second)
+	require.NoErrorf(t, err,
+		"ADR-0007:S4: the daemon did not answer a fence notice, so the provider can never tell it was heard. "+
+			"Fix: subscribe fenceNoticeSubs and respond in handleFenceNotice (daemon_handlers_fence.go)")
+	var ack ebsprovider.FenceNoticeAck
+	require.NoError(t, json.Unmarshal(reply.Data, &ack))
+	return ack
+}
+
+func fenceNoticePayload(t *testing.T, notice ebsprovider.FenceNotice) []byte {
+	t.Helper()
+	payload, err := json.Marshal(notice)
+	require.NoError(t, err)
+	return payload
+}
+
+// memoryProviderPublishing is a non-viperblock provider holding volumeID
+// published to node-a, announcing fences on nc.
+func memoryProviderPublishing(t *testing.T, nc *nats.Conn, volumeID string) *ebsprovider.MemoryProvider {
+	t.Helper()
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
+	provider.NotifyFencesOn(nc)
+	_, err := provider.CreateVolume(t.Context(), ebsprovider.CreateVolumeRequest{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
+	})
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(t.Context(), ebsprovider.PublishVolumeRequest{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: "node-a",
+	})
+	require.NoError(t, err)
+	return provider
 }
 
 // TestS4_FenceNoticeIsAcknowledgedOnlyOnceTheGuestIsStopped enforces ADR-0007:S4
-// on the daemon's half of the fence. viperblockd repeats the notice until it is
+// on the daemon's half of the fence. The provider repeats the notice until it is
 // answered, so the answer is what ends delivery, and it must not come before the
 // guest is out of running, or a lost notice is acknowledged for a guest left up.
 func TestS4_FenceNoticeIsAcknowledgedOnlyOnceTheGuestIsStopped(t *testing.T) {
 	d, guest, nc := fenceDaemon(t, "vol-fencedack")
 
-	payload, err := json.Marshal(viperblocklegacyv1.VolumeFencedEvent{
-		Volume: "vol-fencedack", Node: "node-a", Winner: "node-b", Reason: "volume lease moved to node-b",
-	})
-	require.NoError(t, err)
-
-	reply, err := nc.Request(viperblocklegacyv1.VolumeFencedSubject(d.node), payload, 2*time.Second)
-	require.NoErrorf(t, err,
-		"ADR-0007:S4: the daemon did not answer a fence notice, so viperblockd can never tell it was heard. "+
-			"Fix: respond in handleVolumeFenced (daemon_handlers_fence.go)")
-
-	var ack viperblocklegacyv1.VolumeFencedAck
-	require.NoError(t, json.Unmarshal(reply.Data, &ack))
-	assert.Equal(t, guest.ID, ack.Instance, "the ack names the guest the fence stopped")
-	assert.Empty(t, ack.Error)
+	ack := requestFenceNotice(t, nc, fenceNoticePayload(t, ebsprovider.FenceNotice{
+		Versioned: ebsprovider.NewVersioned(),
+		VolumeID:  "vol-fencedack", NodeID: "node-a", Winner: "node-b", Reason: "volume lease moved to node-b",
+	}))
+	assert.Equal(t, guest.ID, ack.InstanceID, "the ack names the guest the fence stopped")
+	assert.Nil(t, ack.Error)
 	assert.NotEqual(t, vm.StateRunning, d.vmMgr.Status(guest),
 		"ADR-0007:S4: the fence was acknowledged while the guest was still running")
 }
 
+// TestS4_FenceNoticeIsNotAcknowledgedWhileTheStopIsInProgress enforces
+// ADR-0007:S4 on the order of the stop and the ack: with the stop held open,
+// no answer may reach the provider until it completes.
+func TestS4_FenceNoticeIsNotAcknowledgedWhileTheStopIsInProgress(t *testing.T) {
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	var releaseOnce sync.Once
+	t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
+
+	var mgr *vm.Manager
+	mgr = vm.NewManagerWithDeps(vm.Deps{NodeID: "node-a", TransitionState: func(v *vm.VM, target vm.InstanceState) error {
+		close(entered)
+		<-release
+		mgr.Inspect(v, func(v *vm.VM) { v.Status = target })
+		return nil
+	}})
+	_, guest, nc := fenceDaemonWithManager(t, "vol-fenceheld", mgr)
+
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	type answer struct {
+		msg *nats.Msg
+		err error
+	}
+	answers := make(chan answer, 1)
+	payload := fenceNoticePayload(t, ebsprovider.FenceNotice{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-fenceheld", NodeID: "node-a"})
+	go func() {
+		msg, err := nc.Request(subject, payload, 10*time.Second)
+		answers <- answer{msg, err}
+	}()
+
+	select {
+	case <-entered:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the fence never reached the guest's stop, so this test proved nothing about the ack order")
+	}
+	select {
+	case got := <-answers:
+		t.Fatalf("ADR-0007:S4: the fence was acknowledged (%v) before the guest's stop finished. "+
+			"Fix: ack only after FenceVolume returns in handleFenceNotice (daemon_handlers_fence.go)", got.err)
+	case <-time.After(300 * time.Millisecond):
+	}
+
+	releaseOnce.Do(func() { close(release) })
+	var got answer
+	select {
+	case got = <-answers:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the fence was never acknowledged once the stop finished")
+	}
+	require.NoError(t, got.err)
+	var ack ebsprovider.FenceNoticeAck
+	require.NoError(t, json.Unmarshal(got.msg.Data, &ack))
+	assert.Equal(t, guest.ID, ack.InstanceID)
+}
+
+// TestS4_FenceNoticeIsServedByTheProductionSubscriptionTable keeps the
+// subscription in subscribeAll: a handler nothing subscribes is a fence no
+// daemon hears.
+func TestS4_FenceNoticeIsServedByTheProductionSubscriptionTable(t *testing.T) {
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	ns, nc, _ := testutil.StartTestJetStream(t)
+
+	d := newSubscribeTestDaemon(t, nc, "", "")
+	d.node = "node-a"
+	d.vmMgr = vm.NewManagerWithDeps(vm.Deps{NodeID: "node-a"})
+	guest := fencedGuest("vol-fencedsubscribed")
+	d.vmMgr.Insert(guest)
+	require.NoError(t, d.subscribeAll())
+	require.NoError(t, nc.Flush())
+
+	caller, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(caller.Close)
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	reply, err := caller.Request(subject, fenceNoticePayload(t, ebsprovider.FenceNotice{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-fencedsubscribed", NodeID: "node-a",
+	}), 2*time.Second)
+	require.NoErrorf(t, err,
+		"ADR-0007:S4: the running daemon does not serve fence notices. Fix: append fenceNoticeSubs in subscribeAll (daemon.go)")
+	var ack ebsprovider.FenceNoticeAck
+	require.NoError(t, json.Unmarshal(reply.Data, &ack))
+	assert.Nil(t, ack.Error)
+	assert.Equal(t, guest.ID, ack.InstanceID)
+}
+
+// TestNewDaemon_RefusesANodeTheFenceNoticeCannotAddress keeps a daemon from
+// starting where no fence notice can reach it, which would leave a guest
+// running on a revoked export with nothing but a log line to say so.
+func TestNewDaemon_RefusesANodeTheFenceNoticeCannotAddress(t *testing.T) {
+	for _, node := range []string{"", "node.a", "node-*", "node>"} {
+		t.Run(node, func(t *testing.T) {
+			d, err := NewDaemon(&config.ClusterConfig{Node: node, Nodes: map[string]config.Config{node: {}}})
+			require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
+			assert.Contains(t, err.Error(), "cluster config node")
+			assert.Nil(t, d)
+		})
+	}
+}
+
+// TestFenceNotice_SpanAttributesTheAccountAndTheRefusal reads the daemon's
+// span for a refused notice: a server span for the fenced verb, carrying the
+// sender's account and the refusal's code rather than a successful round trip.
+func TestFenceNotice_SpanAttributesTheAccountAndTheRefusal(t *testing.T) {
+	recorder := testutil.RecordSpans(t)
+	_, _, nc := fenceDaemon(t, "vol-unused")
+
+	ctx := ebsprovider.WithAccountID(t.Context(), "123456789012")
+	_, err := ebsprovider.NotifyFenced(ctx, nc, ebsprovider.FenceNotice{
+		Versioned: ebsprovider.Versioned{SchemaVersion: ebsprovider.SchemaVersion + 1}, VolumeID: "vol-unused", NodeID: "node-a",
+	}, 2*time.Second)
+	require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion)
+
+	_, server := testutil.SpansByKind(t, recorder)
+	assert.Equal(t, "ebs.volume.fenced", server.Name())
+	assert.Equal(t, codes.Error, server.Status().Code)
+	assert.Equal(t, string(ebsprovider.ErrorCodeUnsupportedVersion), testutil.SpanAttribute(t, server, "ebs.provider.error_code").AsString())
+	assert.Equal(t, "123456789012", testutil.SpanAttribute(t, server, "aws.account_id").AsString())
+}
+
+// TestFenceNotice_AckEchoesTheNoticeVersion keeps the answer readable by the
+// sender: an ack in a version other than the notice's is one it may refuse.
+func TestFenceNotice_AckEchoesTheNoticeVersion(t *testing.T) {
+	for _, version := range ebsprovider.SupportedSchemaVersions() {
+		_, _, nc := fenceDaemon(t, "vol-versioned")
+		ack := requestFenceNotice(t, nc, fenceNoticePayload(t, ebsprovider.FenceNotice{
+			Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: "vol-versioned", NodeID: "node-a",
+		}))
+		assert.Nil(t, ack.Error)
+		assert.Equalf(t, version, ack.SchemaVersion, "a notice at version %d was answered at another", version)
+	}
+}
+
+// TestS4_FenceNoticeFromAnyProviderStopsTheGuest holds the daemon to the
+// contract rather than to viperblockd: a provider that is not viperblock
+// revoking the export a local guest runs on gets that guest stopped.
+func TestS4_FenceNoticeFromAnyProviderStopsTheGuest(t *testing.T) {
+	d, guest, nc := fenceDaemon(t, "vol-memoryfenced")
+	provider := memoryProviderPublishing(t, nc, "vol-memoryfenced")
+
+	stopped, err := provider.RevokePublication(t.Context(), "vol-memoryfenced", "node-b")
+	require.NoErrorf(t, err,
+		"ADR-0007:S4: the daemon did not take a fence notice from a non-viperblock provider. "+
+			"Fix: subscribe the contract fence-notice subject in fenceNoticeSubs (daemon_handlers_fence.go)")
+	assert.Equal(t, guest.ID, stopped)
+	assert.Equal(t, vm.StateError, d.vmMgr.Status(guest),
+		"ADR-0007:S4: the guest runs on against an export its provider revoked")
+}
+
+// TestFenceNotice_VolumeNoLocalGuestUsesChangesNothing is the ordinary case: a
+// volume fenced after its guest has gone leaves every local guest as it was.
+func TestFenceNotice_VolumeNoLocalGuestUsesChangesNothing(t *testing.T) {
+	d, guest, nc := fenceDaemon(t, "vol-inuse")
+	provider := memoryProviderPublishing(t, nc, "vol-noguest")
+
+	stopped, err := provider.RevokePublication(t.Context(), "vol-noguest", "node-b")
+	require.NoError(t, err, "a fence with nothing to stop is not an error")
+	assert.Empty(t, stopped)
+	assert.Equal(t, vm.StateRunning, d.vmMgr.Status(guest), "a guest on another volume must not be stopped")
+}
+
 // TestS4_RepeatedFenceNoticeIsHarmless covers the cost of the retry: the notice
 // can arrive more than once, and a second copy must leave the fenced guest as the
 // first did rather than re-driving it or failing.
 func TestS4_RepeatedFenceNoticeIsHarmless(t *testing.T) {
 	d, guest, nc := fenceDaemon(t, "vol-fencedtwice")
 
-	payload, err := json.Marshal(viperblocklegacyv1.VolumeFencedEvent{Volume: "vol-fencedtwice", Winner: "node-b"})
-	require.NoError(t, err)
-
+	payload := fenceNoticePayload(t, ebsprovider.FenceNotice{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-fencedtwice", NodeID: "node-a", Winner: "node-b",
+	})
 	for range 2 {
-		reply, err := nc.Request(viperblocklegacyv1.VolumeFencedSubject(d.node), payload, 2*time.Second)
-		require.NoError(t, err)
-		var ack viperblocklegacyv1.VolumeFencedAck
-		require.NoError(t, json.Unmarshal(reply.Data, &ack))
-		assert.Empty(t, ack.Error, "a repeated notice is not an error")
+		ack := requestFenceNotice(t, nc, payload)
+		assert.Nil(t, ack.Error, "a repeated notice is not an error")
 	}
 	assert.Equal(t, vm.StateError, d.vmMgr.Status(guest))
 }
 
-// TestVolumeFenced_BadEventIsAnsweredWithAnError stops a notice that can never be
-// acted on from being repeated forever.
-func TestVolumeFenced_BadEventIsAnsweredWithAnError(t *testing.T) {
-	d, _, nc := fenceDaemon(t, "vol-unused")
-
-	reply, err := nc.Request(viperblocklegacyv1.VolumeFencedSubject(d.node), []byte(`{"volume":""}`), 2*time.Second)
-	require.NoError(t, err)
-	var ack viperblocklegacyv1.VolumeFencedAck
-	require.NoError(t, json.Unmarshal(reply.Data, &ack))
-	assert.NotEmpty(t, ack.Error)
+// TestFenceNotice_UnactionableNoticeIsAnsweredWithAnError stops a notice that
+// can never be acted on from being repeated forever, and leaves the guest alone.
+func TestFenceNotice_UnactionableNoticeIsAnsweredWithAnError(t *testing.T) {
+	cases := map[string]struct {
+		raw  []byte
+		code ebsprovider.ErrorCode
+	}{
+		"undecodable":     {[]byte(`{`), ebsprovider.ErrorCodeInvalidArgument},
+		"no volume":       {[]byte(`{"schema_version":1,"volume_id":""}`), ebsprovider.ErrorCodeInvalidArgument},
+		"unknown version": {[]byte(`{"schema_version":0,"volume_id":"vol-unused"}`), ebsprovider.ErrorCodeUnsupportedVersion},
+		"future version":  {fmt.Appendf(nil, `{"schema_version":%d,"volume_id":"vol-unused"}`, ebsprovider.SchemaVersion+1), ebsprovider.ErrorCodeUnsupportedVersion},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			d, guest, nc := fenceDaemon(t, "vol-unused")
+			ack := requestFenceNotice(t, nc, tc.raw)
+			require.NotNil(t, ack.Error)
+			assert.Equal(t, tc.code, ack.Error.Code)
+			if tc.code == ebsprovider.ErrorCodeUnsupportedVersion {
+				assert.Contains(t, ack.Error.Message, "unsupported schema version ")
+				assert.Contains(t, ack.Error.Message, ebsprovider.SupportedVersionsText())
+			}
+			assert.Equal(t, vm.StateRunning, d.vmMgr.Status(guest))
+		})
+	}
 }

spinifex/runtime/daemon/daemon_handlers_volume.go

@@ -11,7 +11,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
-	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/nats-io/nats.go"
@@ -139,50 +138,6 @@ func (d *Daemon) handleDetachVolume(ctx context.Context, msg *nats.Msg, command
 	return outcomeSuccess
 }
 
-// handleDrainVolume flushes the volume's in-flight writes to S3 by dialing the
-// drain socket the local NBD plugin serves. ec2.CreateSnapshot is answered by
-// any node in the cluster, but only the node hosting the instance subscribes
-// ec2.cmd.{instanceID}, so routing the drain here lands it where the writes are.
-//
-// Runs on its own goroutine (see handleEC2Events) so a long flush cannot hold
-// the instance's command subscription.
-func (d *Daemon) handleDrainVolume(ctx context.Context, msg *nats.Msg, command ec2v1.EC2InstanceCommand, instance *vm.VM) string {
-	ctx, span := startOpSpan(ctx, "ec2.DrainVolume", command.ID)
-	var err error
-	defer func() { endOpSpan(span, err) }()
-
-	if command.DrainVolumeData == nil || command.DrainVolumeData.VolumeID == "" {
-		slog.ErrorContext(ctx, "DrainVolume: missing drain volume data", "instanceId", command.ID)
-		return respondErrorOutcome(d.node, msg, awserrors.ErrorInvalidParameterValue)
-	}
-
-	volumeID := command.DrainVolumeData.VolumeID
-
-	// Only stable post-teardown states prove the volume was sealed. Transitional
-	// states still have a live or shutting-down writer, so they must drain or
-	// fail rather than acknowledge an older checkpoint as current.
-	status := d.vmMgr.Status(instance)
-	if status == vm.StateStopped || status == vm.StateTerminated {
-		slog.InfoContext(ctx, "DrainVolume: instance teardown is complete, nothing to drain",
-			"volumeId", volumeID, "instanceId", command.ID, "status", status)
-		respondWithJSON(d.node, msg, ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusNotRunning})
-		return outcomeSuccess
-	}
-
-	slog.InfoContext(ctx, "Draining volume before snapshot", "volumeId", volumeID, "instanceId", command.ID)
-
-	// A missing or unresponsive socket under a running instance means the writes
-	// cannot be made current, so the caller must fail its snapshot rather than
-	// read a stale checkpoint.
-	if err = ec2snapshot.DrainVolumeSocket(d.config.DataDir, volumeID); err != nil {
-		slog.ErrorContext(ctx, "DrainVolume: drain failed", "volumeId", volumeID, "instanceId", command.ID, "err", err)
-		return respondErrorOutcome(d.node, msg, awserrors.ErrorServerInternal)
-	}
-
-	respondWithJSON(d.node, msg, ec2v1.DrainVolumeResponse{VolumeID: volumeID, Status: ec2v1.DrainVolumeStatusDrained})
-	return outcomeSuccess
-}
-
 // attachDetachErrorCode maps a vm.Manager error returned by AttachVolume
 // or DetachVolume to the AWS API error code that the SDK expects.
 func attachDetachErrorCode(err error) string {

spinifex/runtime/daemon/daemon_handlers_volume_test.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"strings"
 	"testing"
 	"time"
@@ -12,17 +11,14 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
-	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.uber.org/goleak"
 )
 
 // TestAttachDetachErrorCode locks down the manager-error → AWS-API-code
@@ -370,257 +366,6 @@ func TestAttachVolume_IdempotentSameInstance_DeviceMismatch(t *testing.T) {
 	assert.Contains(t, string(resp.Data), awserrors.ErrorVolumeInUse)
 }
 
-// drainCommandFor builds the drain command a snapshot addresses to the node
-// hosting instanceID.
-func drainCommandFor(t *testing.T, instanceID, volumeID string) []byte {
-	t.Helper()
-	command := ec2v1.EC2InstanceCommand{
-		ID:              instanceID,
-		Attributes:      ec2v1.EC2CommandAttributes{DrainVolume: true},
-		DrainVolumeData: &ec2v1.DrainVolumeData{VolumeID: volumeID},
-	}
-	data, err := json.Marshal(command)
-	require.NoError(t, err)
-	return data
-}
-
-// drainTestDaemon returns a daemon owning a running instance, with a data dir
-// short enough to hold the drain socket path.
-func drainTestDaemon(t *testing.T, instanceID string) *Daemon {
-	t.Helper()
-	return drainTestDaemonInState(t, instanceID, vm.StateRunning)
-}
-
-// drainTestDaemonInState is drainTestDaemon for an instance the node still
-// holds in some other state.
-func drainTestDaemonInState(t *testing.T, instanceID string, status vm.InstanceState) *Daemon {
-	t.Helper()
-	daemon := createTestDaemon(t, sharedNATSURL)
-	daemon.config.DataDir = testutil.SocketTempDir(t)
-	daemon.vmMgr.Insert(&vm.VM{
-		ID:           instanceID,
-		Status:       status,
-		AccountID:    testAccountID,
-		InstanceType: getTestInstanceType(t),
-		Instance:     &ec2.Instance{},
-	})
-	return daemon
-}
-
-// drainRequest issues a drain on the instance's command subject, returning the
-// raw reply.
-func drainRequest(t *testing.T, daemon *Daemon, instanceID, volumeID string) *nats.Msg {
-	t.Helper()
-	reply, err := sendDrainCommand(daemon, instanceID, drainCommandFor(t, instanceID, volumeID))
-	require.NoError(t, err)
-	return reply
-}
-
-// sendDrainCommand is drainRequest without the assertions, so a test can issue
-// one from a goroutine and assert on the result back on the test goroutine.
-func sendDrainCommand(daemon *Daemon, instanceID string, command []byte) (*nats.Msg, error) {
-	msg := nats.NewMsg("ec2.cmd." + instanceID)
-	msg.Data = command
-	msg.Header.Set(natsmsg.AccountIDHeader, testAccountID)
-	return daemon.natsConn.RequestMsg(msg, 5*time.Second)
-}
-
-// subscribeEC2Commands wires the daemon's real dispatcher onto the per-instance
-// command subject, so tests exercise the same serial delivery production has.
-func subscribeEC2Commands(t *testing.T, daemon *Daemon, instanceID string) {
-	t.Helper()
-	sub, err := daemon.natsConn.Subscribe("ec2.cmd."+instanceID, daemon.handleEC2Events)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-}
-
-// A drain routed to the node hosting the instance reaches the local socket and
-// acks, which is what lets a snapshot answered elsewhere read a current
-// checkpoint.
-func TestDrainVolume_HostNodeAcksLocalSocket(t *testing.T) {
-	const instanceID, volumeID = "i-drain-ok", "vol-drain-ok"
-	daemon := drainTestDaemon(t, instanceID)
-	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "OK\n")
-
-	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-		testAccountID, drainCommandFor(t, instanceID, volumeID))
-
-	var ack ec2v1.DrainVolumeResponse
-	require.NoError(t, json.Unmarshal(resp.Data, &ack))
-	assert.Equal(t, volumeID, ack.VolumeID)
-	assert.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
-}
-
-// The owning node having no socket for the volume means the writes cannot be
-// flushed. That must reach the caller as an error, never as a silent success:
-// the caller would otherwise snapshot a stale checkpoint.
-func TestDrainVolume_HostNodeWithoutSocketFails(t *testing.T) {
-	const instanceID, volumeID = "i-drain-nosock", "vol-drain-nosock"
-	daemon := drainTestDaemon(t, instanceID)
-
-	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-		testAccountID, drainCommandFor(t, instanceID, volumeID))
-
-	assert.Equal(t, awserrors.ErrorServerInternal, replyErrCode(t, resp.Data))
-}
-
-// A plugin that refuses the drain (ERR rather than OK) is a failure, not an ack.
-func TestDrainVolume_HostNodeRelaysSocketFailure(t *testing.T) {
-	const instanceID, volumeID = "i-drain-err", "vol-drain-err"
-	daemon := drainTestDaemon(t, instanceID)
-	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "ERR\n")
-
-	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-		testAccountID, drainCommandFor(t, instanceID, volumeID))
-
-	assert.Equal(t, awserrors.ErrorServerInternal, replyErrCode(t, resp.Data))
-}
-
-// A drain command with no volume is a caller error, not a server fault.
-func TestDrainVolume_MissingVolumeDataIsInvalidParameter(t *testing.T) {
-	const instanceID = "i-drain-novol"
-	daemon := drainTestDaemon(t, instanceID)
-
-	command := ec2v1.EC2InstanceCommand{
-		ID:         instanceID,
-		Attributes: ec2v1.EC2CommandAttributes{DrainVolume: true},
-	}
-	data, err := json.Marshal(command)
-	require.NoError(t, err)
-
-	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-		testAccountID, data)
-
-	assert.Equal(t, awserrors.ErrorInvalidParameterValue, replyErrCode(t, resp.Data))
-}
-
-// Only the instance's owner may drain its volume: the command carries the same
-// ownership gate as every other per-instance command.
-func TestDrainVolume_ForeignAccountRejected(t *testing.T) {
-	const instanceID, volumeID = "i-drain-foreign", "vol-drain-foreign"
-	daemon := drainTestDaemon(t, instanceID)
-	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "OK\n")
-
-	resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-		"999988887777", drainCommandFor(t, instanceID, volumeID))
-
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, replyErrCode(t, resp.Data))
-}
-
-// Only stable post-teardown states prove the volume is sealed. These states
-// acknowledge not-running without touching the absent plugin socket.
-func TestDrainVolume_CompletedTeardownAcksNotRunning(t *testing.T) {
-	for _, status := range []vm.InstanceState{vm.StateStopped, vm.StateTerminated} {
-		t.Run(string(status), func(t *testing.T) {
-			instanceID := "i-drain-" + string(status)
-			volumeID := "vol-drain-" + string(status)
-			daemon := drainTestDaemonInState(t, instanceID, status)
-
-			resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-				testAccountID, drainCommandFor(t, instanceID, volumeID))
-
-			var ack ec2v1.DrainVolumeResponse
-			require.NoError(t, json.Unmarshal(resp.Data, &ack))
-			assert.Equal(t, volumeID, ack.VolumeID)
-			assert.Equal(t, ec2v1.DrainVolumeStatusNotRunning, ack.Status)
-		})
-	}
-}
-
-// Transitional and uncertain states are not proof of a sealed checkpoint. An
-// absent socket in any of them must fail rather than silently snapshot stale
-// data while launch or teardown may still own the volume.
-func TestDrainVolume_UnsealedStatesRequireDrain(t *testing.T) {
-	statuses := []vm.InstanceState{
-		vm.StateRunning,
-		vm.StateStopping,
-		vm.StateShuttingDown,
-		vm.StatePending,
-		vm.StateProvisioning,
-		vm.StateError,
-	}
-	for _, status := range statuses {
-		t.Run(string(status), func(t *testing.T) {
-			instanceID := "i-drain-" + string(status)
-			volumeID := "vol-drain-" + string(status)
-			daemon := drainTestDaemonInState(t, instanceID, status)
-
-			resp := requestHandler(t, daemon.natsConn, "ec2.cmd."+instanceID, daemon.handleEC2Events,
-				testAccountID, drainCommandFor(t, instanceID, volumeID))
-
-			assert.Equal(t, awserrors.ErrorServerInternal, replyErrCode(t, resp.Data))
-		})
-	}
-}
-
-// nats.go delivers a subscription's messages serially, so a drain run inline
-// would hold ec2.cmd.{instanceID} for the length of the flush — stalling stop,
-// terminate and hot-plug for that instance. A drain still flushing must not
-// keep the next command waiting.
-func TestDrainVolume_SlowDrainDoesNotBlockTheCommandSubject(t *testing.T) {
-	const instanceID = "i-drain-parallel"
-	daemon := drainTestDaemon(t, instanceID)
-	accepted, release := testutil.StartBlockingDrainSocket(t, daemon.config.DataDir, "vol-drain-slow", "OK\n")
-	testutil.StartDrainSocket(t, daemon.config.DataDir, "vol-drain-fast", "OK\n")
-	subscribeEC2Commands(t, daemon, instanceID)
-
-	type reply struct {
-		msg *nats.Msg
-		err error
-	}
-	slowCommand := drainCommandFor(t, instanceID, "vol-drain-slow")
-	slow := make(chan reply, 1)
-	go func() {
-		msg, err := sendDrainCommand(daemon, instanceID, slowCommand)
-		slow <- reply{msg, err}
-	}()
-
-	// Only assert once the slow drain is genuinely mid-flush.
-	select {
-	case <-accepted:
-	case <-time.After(5 * time.Second):
-		t.Fatal("the slow drain never reached its socket")
-	}
-
-	fast := drainRequest(t, daemon, instanceID, "vol-drain-fast")
-	var ack ec2v1.DrainVolumeResponse
-	require.NoError(t, json.Unmarshal(fast.Data, &ack))
-	assert.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
-
-	release()
-	select {
-	case got := <-slow:
-		require.NoError(t, got.err)
-		require.NoError(t, json.Unmarshal(got.msg.Data, &ack))
-		assert.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
-	case <-time.After(5 * time.Second):
-		t.Fatal("the released drain never replied")
-	}
-}
-
-// Running the drain off the delivery goroutine must not leak it: the goroutine
-// has to be gone once the ack is on the wire, or a snapshot loop against a
-// wedged plugin accumulates them.
-func TestDrainVolume_DispatchedGoroutineDoesNotLeak(t *testing.T) {
-	const instanceID, volumeID = "i-drain-leak", "vol-drain-leak"
-	daemon := drainTestDaemon(t, instanceID)
-	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "OK\n")
-	subscribeEC2Commands(t, daemon, instanceID)
-
-	// Take the baseline after one full round trip: the first request is what
-	// makes nats.go stand up its long-lived response-inbox goroutine, which
-	// would otherwise read as the leak.
-	var ack ec2v1.DrainVolumeResponse
-	require.NoError(t, json.Unmarshal(drainRequest(t, daemon, instanceID, volumeID).Data, &ack))
-	require.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
-	ignoreExisting := goleak.IgnoreCurrent()
-
-	require.NoError(t, json.Unmarshal(drainRequest(t, daemon, instanceID, volumeID).Data, &ack))
-	require.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
-
-	goleak.VerifyNone(t, ignoreExisting)
-}
-
 // seedProviderVolume allocates volumeID in the daemon's EBS provider so
 // operations that reach the provider (expand, delete) find it there.
 func seedProviderVolume(t *testing.T, daemon *Daemon, volumeID string, sizeGiB int64) {

spinifex/runtime/daemon/daemon_test.go

@@ -50,6 +50,7 @@ import (
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	elbv2domain "github.com/mulgadc/spinifex/spinifex/domains/elbv2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
@@ -137,10 +138,10 @@ func createTestDaemon(t *testing.T, natsURL string) *Daemon {
 	// Wire the minimum vm.Deps that handler tests rely on. Lifecycle (Run/Start/
 	// Stop/Terminate) tests still set up their own deps; this gives the
 	// AttachVolume / DetachVolume manager methods enough plumbing to drive
-	// ebs.mount/unmount over NATS using the test's connection.
+	// publish/unpublish over NATS using the test's connection.
 	daemon.vmMgr.SetDeps(vm.Deps{
 		NodeID:               daemon.node,
-		VolumeMounter:        newVolumeMounterAdapter(daemon.natsConn, daemon.node, daemon.volumeService),
+		VolumeMounter:        newNATSMounter(daemon.natsConn, daemon.node, daemon.volumeService),
 		VolumeStateUpdater:   daemon.volumeService,
 		DetachDelay:          daemon.detachDelay,
 		DeviceDeletedTimeout: daemon.deviceDeletedTimeout,
@@ -2352,18 +2353,13 @@ func TestDetachVolume_SuccessPath(t *testing.T) {
 	}
 	daemon.vmMgr.Insert(instance)
 
-	// Subscribe a mock ebs.unmount handler
+	// Stand in for the provider's unpublish.
 	ebsUnmountCalled := make(chan string, 1)
-	ebsSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		var req viperblocklegacyv1.EBSRequest
-		json.Unmarshal(msg.Data, &req)
-		ebsUnmountCalled <- req.Name
-		resp := viperblocklegacyv1.EBSUnMountResponse{Volume: req.Name, Mounted: false}
-		data, _ := json.Marshal(resp)
-		msg.Respond(data)
+	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
+	fake.setUnpublish(func(volumeID string) ebsprovider.UnpublishVolumeResponse {
+		ebsUnmountCalled <- volumeID
+		return ebsprovider.UnpublishVolumeResponse{}
 	})
-	require.NoError(t, err)
-	defer ebsSub.Unsubscribe()
 
 	sub, err := daemon.natsConn.Subscribe(
 		fmt.Sprintf("ec2.cmd.%s", instanceID),
@@ -2404,12 +2400,12 @@ func TestDetachVolume_SuccessPath(t *testing.T) {
 	assert.Equal(t, []string{"device_del", "blockdev-del", "object-del"}, qmpCommands)
 	mu.Unlock()
 
-	// Verify ebs.unmount was called
+	// Verify unpublish was called
 	select {
 	case unmountedVol := <-ebsUnmountCalled:
 		assert.Equal(t, volumeID, unmountedVol)
 	case <-time.After(5 * time.Second):
-		t.Fatal("ebs.unmount was not called")
+		t.Fatal("unpublish was not called")
 	}
 
 	// Verify volume removed from EBSRequests
@@ -2493,14 +2489,7 @@ func TestDetachVolume_ForceFlag(t *testing.T) {
 	}
 	daemon.vmMgr.Insert(instance)
 
-	// Mock ebs.unmount
-	ebsSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		resp := viperblocklegacyv1.EBSUnMountResponse{Mounted: false}
-		data, _ := json.Marshal(resp)
-		msg.Respond(data)
-	})
-	require.NoError(t, err)
-	defer ebsSub.Unsubscribe()
+	serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
 
 	sub, err := daemon.natsConn.Subscribe(
 		fmt.Sprintf("ec2.cmd.%s", instanceID),
@@ -2697,13 +2686,7 @@ func TestDetachVolume_SuccessWithDeviceMatch(t *testing.T) {
 	}
 	daemon.vmMgr.Insert(instance)
 
-	ebsSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		resp := viperblocklegacyv1.EBSUnMountResponse{Mounted: false}
-		data, _ := json.Marshal(resp)
-		msg.Respond(data)
-	})
-	require.NoError(t, err)
-	defer ebsSub.Unsubscribe()
+	serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
 
 	sub, err := daemon.natsConn.Subscribe(
 		fmt.Sprintf("ec2.cmd.%s", instanceID),
@@ -2773,14 +2756,9 @@ func TestAttachVolume_ReplacesStaleEBSRequest(t *testing.T) {
 	}
 	daemon.vmMgr.Insert(instance)
 
-	// Mock ebs.mount to return success with a new NBDURI
-	ebsSub, err := daemon.natsConn.Subscribe("ebs.node-1.mount", func(msg *nats.Msg) {
-		resp := viperblocklegacyv1.EBSMountResponse{URI: "nbd://new:2222"}
-		data, _ := json.Marshal(resp)
-		msg.Respond(data)
-	})
-	require.NoError(t, err)
-	defer ebsSub.Unsubscribe()
+	// The provider publishes the volume with a new NBD URI.
+	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
+	fake.setPublish(func(string) ebsprovider.PublishVolumeResponse { return publishedNBD("nbd://new:2222") })
 
 	sub, err := daemon.natsConn.Subscribe(
 		fmt.Sprintf("ec2.cmd.%s", instanceID),
@@ -3335,95 +3313,33 @@ func TestNewDaemon_WalDirPreservedIfSet(t *testing.T) {
 	assert.Equal(t, "/fast-ssd/wal", d.config.WalDir)
 }
 
-// TestVolumeMounterAdapter_UnmountOne_Success verifies that the adapter's
-// UnmountOne sends an ebs.unmount NATS request and handles a successful
-// response. UnmountOne is shared by the AttachVolume rollback path and the
-// DetachVolume ebs.unmount step.
-func TestVolumeMounterAdapter_UnmountOne_Success(t *testing.T) {
-	natsURL := sharedNATSURL
-
-	daemon := createTestDaemon(t, natsURL)
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
-
-	unmountCalled := make(chan string, 1)
-
-	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		var req viperblocklegacyv1.EBSRequest
-		json.Unmarshal(msg.Data, &req)
-		unmountCalled <- req.Name
-		resp := viperblocklegacyv1.EBSUnMountResponse{Volume: req.Name, Mounted: false}
-		data, _ := json.Marshal(resp)
-		msg.Respond(data)
-	})
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{
-		Name:       "vol-rollback-test",
-		DeviceName: "/dev/sdf",
-	})
-
-	select {
-	case volName := <-unmountCalled:
-		assert.Equal(t, "vol-rollback-test", volName)
-	case <-time.After(2 * time.Second):
-		t.Fatal("ebs.unmount was not called")
-	}
-}
-
 // TestVolumeMounterAdapter_UnmountOne_UnmountError verifies that UnmountOne
-// propagates an error response from ebs.unmount. DetachVolume gates the volume's
-// available transition on this error, so it must not be swallowed.
+// propagates an unpublish refusal. DetachVolume gates the volume's available
+// transition on this error, so it must not be swallowed.
 func TestVolumeMounterAdapter_UnmountOne_UnmountError(t *testing.T) {
-	natsURL := sharedNATSURL
-
-	daemon := createTestDaemon(t, natsURL)
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
-
-	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		resp := viperblocklegacyv1.EBSUnMountResponse{Error: "unmount failed: device busy"}
-		data, _ := json.Marshal(resp)
-		msg.Respond(data)
+	daemon := createTestDaemon(t, sharedNATSURL)
+	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
+	fake.setUnpublish(func(string) ebsprovider.UnpublishVolumeResponse {
+		return unpublishRefusal(ebsprovider.ErrorCodeInternal, "seal volume: device busy")
 	})
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
+	adapter := newNATSMounter(daemon.natsConn, daemon.node, nil)
 
-	require.Error(t, adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-rollback-err"}),
-		"an ebs.unmount error response must propagate so DetachVolume can keep the volume attached")
-}
-
-// TestVolumeMounterAdapter_UnmountOne_StillMounted verifies that UnmountOne
-// returns an error when the unmount response reports the volume still mounted.
-func TestVolumeMounterAdapter_UnmountOne_StillMounted(t *testing.T) {
-	natsURL := sharedNATSURL
-
-	daemon := createTestDaemon(t, natsURL)
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
-
-	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		resp := viperblocklegacyv1.EBSUnMountResponse{Mounted: true}
-		data, _ := json.Marshal(resp)
-		msg.Respond(data)
-	})
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	require.Error(t, adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-still-mounted"}),
-		"a still-mounted response must propagate as an error")
+	err := adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-rollback-err"})
+	require.Error(t, err, "an unpublish refusal must propagate so DetachVolume can keep the volume attached")
+	assert.Contains(t, err.Error(), "device busy")
 }
 
 // TestVolumeMounterAdapter_UnmountOne_RequestFailure verifies that UnmountOne
-// surfaces a failed ebs.unmount NATS request as an error. A closed connection
-// fails immediately, rather than blocking on the full unmountSealTimeout (120s)
-// that a no-subscriber timeout would incur in every preflight run.
+// surfaces a failed unpublish request as an error. A closed connection fails
+// immediately rather than waiting out the provider's seal budget.
 func TestVolumeMounterAdapter_UnmountOne_RequestFailure(t *testing.T) {
 	nc, err := nats.Connect(sharedNATSURL)
 	require.NoError(t, err)
 	nc.Close()
 
-	adapter := newVolumeMounterAdapter(nc, "node-1", nil)
-	require.Error(t, adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-timeout"}),
-		"a failed ebs.unmount request must propagate as an error")
+	adapter := newNATSMounter(nc, "node-1", nil)
+	require.ErrorIs(t, adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-timeout"}), nats.ErrConnectionClosed,
+		"a failed unpublish request must propagate as an error")
 }
 
 // recordingVolState records UpdateVolumeState calls so the bulk-unmount test can
@@ -3469,20 +3385,14 @@ var _ vm.VolumeStateUpdater = (*recordingVolState)(nil)
 func TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
 	volState := &recordingVolState{}
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, volState)
-
-	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		var req viperblocklegacyv1.EBSRequest
-		json.Unmarshal(msg.Data, &req)
-		resp := viperblocklegacyv1.EBSUnMountResponse{Volume: req.Name, Mounted: false}
-		if req.Name == "vol-fail" {
-			resp.Error = "seal volume: predastore unreachable"
+	adapter := newNATSMounter(daemon.natsConn, daemon.node, volState)
+	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
+	fake.setUnpublish(func(volumeID string) ebsprovider.UnpublishVolumeResponse {
+		if volumeID == "vol-fail" {
+			return unpublishRefusal(ebsprovider.ErrorCodeInternal, "seal volume: predastore unreachable")
 		}
-		data, _ := json.Marshal(resp)
-		msg.Respond(data)
+		return ebsprovider.UnpublishVolumeResponse{}
 	})
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
 
 	inst := &vm.VM{
 		ID: "i-1",
@@ -3490,7 +3400,7 @@ func TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable(t *testing.T) {
 			Requests: []viperblocklegacyv1.EBSRequest{{Name: "vol-fail"}, {Name: "vol-ok"}},
 		},
 	}
-	err = adapter.Unmount(t.Context(), inst)
+	err := adapter.Unmount(t.Context(), inst)
 	require.Error(t, err, "bulk Unmount must report a per-volume seal failure")
 	assert.Contains(t, err.Error(), "vol-fail", "the error must name the volume that did not seal")
 
@@ -3501,29 +3411,18 @@ func TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable(t *testing.T) {
 		"a successful seal must still transition the volume to available")
 }
 
-// TestVolumeMounterAdapter_Unmount_NotFoundFlipsToAvailable verifies the
-// timeout-then-completed-seal retry path: an ebs.unmount response reporting
-// NotFound (the seal already completed on a prior, client-timed-out request)
-// must still flip a non-boot data volume to available, while the boot/EFI
-// gate stays intact.
+// TestVolumeMounterAdapter_Unmount_NotFoundFlipsToAvailable verifies that an
+// unpublish answered not_found, which means the provider holds no export of
+// the volume, still flips a non-boot data volume to available, while the
+// boot/EFI gate stays intact.
 func TestVolumeMounterAdapter_Unmount_NotFoundFlipsToAvailable(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
 	volState := &recordingVolState{}
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, volState)
-
-	sub, err := daemon.natsConn.Subscribe(viperblocklegacyv1.UnmountSubject(daemon.node), func(msg *nats.Msg) {
-		var req viperblocklegacyv1.EBSRequest
-		json.Unmarshal(msg.Data, &req)
-		resp := viperblocklegacyv1.EBSUnMountResponse{
-			Volume:   req.Name,
-			NotFound: true,
-			Error:    fmt.Sprintf("Volume %s not found", req.Name),
-		}
-		data, _ := json.Marshal(resp)
-		msg.Respond(data)
+	adapter := newNATSMounter(daemon.natsConn, daemon.node, volState)
+	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
+	fake.setUnpublish(func(volumeID string) ebsprovider.UnpublishVolumeResponse {
+		return unpublishRefusal(ebsprovider.ErrorCodeNotFound, fmt.Sprintf("Volume %s not found", volumeID))
 	})
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
 
 	inst := &vm.VM{
 		ID:        "i-unmount-retry",
@@ -3536,130 +3435,86 @@ func TestVolumeMounterAdapter_Unmount_NotFoundFlipsToAvailable(t *testing.T) {
 		},
 	}
 	require.NoError(t, adapter.Unmount(t.Context(), inst))
+	require.NoError(t, adapter.UnmountOne(t.Context(), "000000000042", viperblocklegacyv1.EBSRequest{Name: "vol-data-retry"}),
+		"a hot detach of a volume the provider no longer holds is already done")
 
 	calls := volState.snapshot()
 	assert.Contains(t, calls, "vol-data-retry:available",
-		"a NotFound response (already-completed seal) must still flip a non-boot data volume to available")
+		"a not_found unpublish must still flip a non-boot data volume to available")
 	assert.NotContains(t, calls, "vol-boot-retry:available",
-		"a boot volume must stay attached even when the seal retry reports NotFound")
+		"a boot volume must stay attached even when the unpublish reports not_found")
 	assert.Equal(t, []string{"000000000042"}, volState.accountsSnapshot(),
 		"the state write must be keyed on the instance's own account, not its ID")
 }
 
-// TestUnmountResponseError covers the shared seal-result parser used by both the
-// gated DetachVolume path (unmountOne) and the tolerated teardown path (Unmount).
-func TestUnmountResponseError(t *testing.T) {
-	t.Run("success (not mounted, no error)", func(t *testing.T) {
-		data, _ := json.Marshal(viperblocklegacyv1.EBSUnMountResponse{Volume: "v", Mounted: false})
-		require.NoError(t, unmountResponseError(data))
-	})
-	t.Run("seal error propagates", func(t *testing.T) {
-		data, _ := json.Marshal(viperblocklegacyv1.EBSUnMountResponse{Volume: "v", Error: "seal volume: boom"})
-		require.Error(t, unmountResponseError(data))
-	})
-	t.Run("still mounted propagates", func(t *testing.T) {
-		data, _ := json.Marshal(viperblocklegacyv1.EBSUnMountResponse{Volume: "v", Mounted: true})
-		require.Error(t, unmountResponseError(data))
-	})
-	t.Run("malformed payload propagates", func(t *testing.T) {
-		require.Error(t, unmountResponseError([]byte("not json")))
-	})
-	t.Run("not found treated as idempotent success", func(t *testing.T) {
-		data, _ := json.Marshal(viperblocklegacyv1.EBSUnMountResponse{Volume: "v", NotFound: true, Error: "Volume v not found"})
-		require.NoError(t, unmountResponseError(data),
-			"a NotFound response means the seal already completed on a prior request; a timeout-then-retry must not be treated as a failure")
+// TestVolumeMounterAdapter_Mount_PartialFailureRollback verifies that when
+// the second of three volumes is refused, the adapter unpublishes the volume
+// it already published in the same Mount() call and nothing else. Without
+// rollback, a launch failure would leave the first volume's export live.
+func TestVolumeMounterAdapter_Mount_PartialFailureRollback(t *testing.T) {
+	nc := connectMounterTest(t)
+	node := newMounterTestNode()
+	fake := serveFakeNodeProvider(t, nc, node)
+	fake.setPublish(func(volumeID string) ebsprovider.PublishVolumeResponse {
+		if volumeID == "vol-2" {
+			return ebsprovider.PublishVolumeResponse{Error: providerRefusal(ebsprovider.ErrorCodeInternal, "simulated mount failure")}
+		}
+		return publishedNBD("nbd://mounted-" + volumeID)
 	})
+	adapter := newNATSMounter(nc, node, nil)
+
+	err := adapter.Mount(t.Context(), mounterTestVM("vol-1", "vol-2", "vol-3"))
+	require.Error(t, err, "Mount should propagate the vol-2 failure")
+	assert.Contains(t, err.Error(), "simulated mount failure")
+	assert.Equal(t, []string{"vol-1"}, fake.unpublished(),
+		"only the previously published vol-1 is rolled back; a refused publish exported nothing")
 }
 
-// TestVolumeMounterAdapter_Mount_PartialFailureRollback verifies that when
-// any of Mount's per-volume failure paths fires mid-fan-out, the adapter
-// unmounts the volumes already successfully mounted in the same Mount()
-// call. Each subtest exercises a distinct rollback trigger so a regression
-// that drops rollback() from one branch (mount-response error, malformed
-// response, NATS layer failure) is caught individually. Without rollback,
-// a launch failure on the second of three volumes would leave the first
-// volume's viperblockd attached and the NBD socket live, leaking
-// resources every retry.
-func TestVolumeMounterAdapter_Mount_PartialFailureRollback(t *testing.T) {
-	tests := []struct {
-		name        string
-		respondVol2 func(msg *nats.Msg)
-		wantErrSub  string
-	}{
-		{
-			name: "MountResponseError",
-			respondVol2: func(msg *nats.Msg) {
-				resp := viperblocklegacyv1.EBSMountResponse{Error: "simulated mount failure"}
-				data, _ := json.Marshal(resp)
-				msg.Respond(data)
-			},
-			wantErrSub: "simulated mount failure",
-		},
-		{
-			name: "MalformedResponse",
-			respondVol2: func(msg *nats.Msg) {
-				msg.Respond([]byte("not-valid-json"))
-			},
-			wantErrSub: "invalid character",
-		},
-	}
+// A publish reply that cannot be decoded may still have started an export, so
+// the failing volume is torn down along with the one already published.
+func TestVolumeMounterAdapter_Mount_UndecodableReplyRollsBackBoth(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+	adapter := newNATSMounter(daemon.natsConn, daemon.node, nil)
 
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			daemon := createTestDaemon(t, sharedNATSURL)
-			adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+	publishSubject, err := ebsprovider.PublishSubject(daemon.node)
+	require.NoError(t, err)
+	mountSub, err := daemon.natsConn.Subscribe(publishSubject, func(msg *nats.Msg) {
+		var req ebsprovider.PublishVolumeRequest
+		require.NoError(t, json.Unmarshal(msg.Data, &req))
+		if req.VolumeID == "vol-2" {
+			msg.Respond([]byte("not-valid-json"))
+			return
+		}
+		resp := publishedNBD("nbd://mounted-" + req.VolumeID)
+		resp.Versioned = ebsprovider.ReplyVersioned(req.SchemaVersion)
+		data, _ := json.Marshal(resp)
+		msg.Respond(data)
+	})
+	require.NoError(t, err)
+	defer mountSub.Unsubscribe()
 
-			mountSub, err := daemon.natsConn.Subscribe("ebs.node-1.mount", func(msg *nats.Msg) {
-				var req viperblocklegacyv1.EBSRequest
-				require.NoError(t, json.Unmarshal(msg.Data, &req))
-				if req.Name == "vol-2" {
-					tt.respondVol2(msg)
-					return
-				}
-				resp := viperblocklegacyv1.EBSMountResponse{URI: "nbd://mounted-" + req.Name}
-				data, _ := json.Marshal(resp)
-				msg.Respond(data)
-			})
-			require.NoError(t, err)
-			defer mountSub.Unsubscribe()
+	unmounted := make(chan string, 3)
+	unpublishSubject, err := ebsprovider.UnpublishSubject(daemon.node)
+	require.NoError(t, err)
+	unmountSub, err := daemon.natsConn.Subscribe(unpublishSubject, func(msg *nats.Msg) {
+		var req ebsprovider.UnpublishVolumeRequest
+		require.NoError(t, json.Unmarshal(msg.Data, &req))
+		unmounted <- req.VolumeID
+		data, _ := json.Marshal(ebsprovider.UnpublishVolumeResponse{Versioned: ebsprovider.ReplyVersioned(req.SchemaVersion)})
+		msg.Respond(data)
+	})
+	require.NoError(t, err)
+	defer unmountSub.Unsubscribe()
 
-			unmounted := make(chan string, 3)
-			unmountSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-				var req viperblocklegacyv1.EBSRequest
-				require.NoError(t, json.Unmarshal(msg.Data, &req))
-				unmounted <- req.Name
-				resp := viperblocklegacyv1.EBSUnMountResponse{Volume: req.Name, Mounted: false}
-				data, _ := json.Marshal(resp)
-				msg.Respond(data)
-			})
-			require.NoError(t, err)
-			defer unmountSub.Unsubscribe()
+	err = adapter.Mount(t.Context(), mounterTestVM("vol-1", "vol-2", "vol-3"))
+	require.ErrorIs(t, err, vm.ErrMountAmbiguous)
+	require.ErrorIs(t, err, ebsprovider.ErrUnreadableReply)
+	assert.Contains(t, err.Error(), "invalid character")
 
-			instance := &vm.VM{
-				ID:        "i-mount-rollback",
-				AccountID: testAccountID,
-				EBSRequests: vm.EBSRequests{
-					Requests: []viperblocklegacyv1.EBSRequest{
-						{Name: "vol-1"},
-						{Name: "vol-2"},
-						{Name: "vol-3"},
-					},
-				},
-			}
-
-			err = adapter.Mount(t.Context(), instance)
-			require.Error(t, err, "Mount should propagate the vol-2 failure")
-			assert.Contains(t, err.Error(), tt.wantErrSub)
-
-			// Mount returns only after rollback completes (the unmount NATS
-			// round-trip is synchronous), and the unmount subscriber sends on
-			// the buffered channel before responding — so by the time Mount
-			// returns, every rollback unmount is observable in `unmounted`.
-			require.Len(t, unmounted, 1,
-				"exactly one volume (vol-1, the previously mounted one) should be rolled back")
-			assert.Equal(t, "vol-1", <-unmounted)
-		})
-	}
+	// Mount returns only after rollback completes, and the subscriber sends on
+	// the buffered channel before responding, so every rollback is seen here.
+	require.Len(t, unmounted, 2, "vol-1 and the ambiguous vol-2 are rolled back")
+	assert.ElementsMatch(t, []string{"vol-1", "vol-2"}, []string{<-unmounted, <-unmounted})
 }
 
 // TestVolumeMounterAdapter_Mount_RollbackFailurePropagates verifies that
@@ -3668,31 +3523,17 @@ func TestVolumeMounterAdapter_Mount_PartialFailureRollback(t *testing.T) {
 // attached without surfacing the leak to the caller.
 func TestVolumeMounterAdapter_Mount_RollbackFailurePropagates(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
-
-	mountSub, err := daemon.natsConn.Subscribe("ebs.node-1.mount", func(msg *nats.Msg) {
-		var req viperblocklegacyv1.EBSRequest
-		require.NoError(t, json.Unmarshal(msg.Data, &req))
-		if req.Name == "vol-2" {
-			resp := viperblocklegacyv1.EBSMountResponse{Error: "primary mount failure"}
-			data, _ := json.Marshal(resp)
-			msg.Respond(data)
-			return
+	adapter := newNATSMounter(daemon.natsConn, daemon.node, nil)
+	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
+	fake.setPublish(func(volumeID string) ebsprovider.PublishVolumeResponse {
+		if volumeID == "vol-2" {
+			return ebsprovider.PublishVolumeResponse{Error: providerRefusal(ebsprovider.ErrorCodeInternal, "primary mount failure")}
 		}
-		resp := viperblocklegacyv1.EBSMountResponse{URI: "nbd://mounted-" + req.Name}
-		data, _ := json.Marshal(resp)
-		msg.Respond(data)
+		return publishedNBD("nbd://mounted-" + volumeID)
 	})
-	require.NoError(t, err)
-	defer mountSub.Unsubscribe()
-
-	unmountSub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
-		resp := viperblocklegacyv1.EBSUnMountResponse{Error: "rollback unmount failed"}
-		data, _ := json.Marshal(resp)
-		msg.Respond(data)
+	fake.setUnpublish(func(string) ebsprovider.UnpublishVolumeResponse {
+		return unpublishRefusal(ebsprovider.ErrorCodeInternal, "rollback unmount failed")
 	})
-	require.NoError(t, err)
-	defer unmountSub.Unsubscribe()
 
 	instance := &vm.VM{
 		ID:        "i-rollback-failure",
@@ -3705,7 +3546,7 @@ func TestVolumeMounterAdapter_Mount_RollbackFailurePropagates(t *testing.T) {
 		},
 	}
 
-	err = adapter.Mount(t.Context(), instance)
+	err := adapter.Mount(t.Context(), instance)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "primary mount failure",
 		"original mount error must be preserved")

spinifex/runtime/daemon/instance_recovery_coverage_test.go

@@ -251,7 +251,7 @@ func TestRecoveryAttemptSucceeds(t *testing.T) {
 	// starting QEMU, which is the one part of a recovery this cannot do.
 	d.vmMgr.SetDeps(vm.Deps{
 		NodeID:             d.node,
-		VolumeMounter:      newVolumeMounterAdapter(d.natsConn, d.node, d.volumeService),
+		VolumeMounter:      newNATSMounter(d.natsConn, d.node, d.volumeService),
 		VolumeStateUpdater: d.volumeService,
 		Hooks: vm.ManagerHooks{BeforeInstanceRelaunch: func(v *vm.VM) error {
 			d.vmMgr.Inspect(v, func(v *vm.VM) { v.Status = vm.StateShuttingDown })

spinifex/runtime/daemon/local_state.go

@@ -42,16 +42,10 @@ type LocalState struct {
 // LocalStatePath returns the absolute path to the per-node instance state file.
 // Empty dataDir falls back to DefaultDataDir.
 func LocalStatePath(dataDir string) string {
-	return filepath.Join(DataDirOrDefault(dataDir), DefaultLocalStateDir, LocalStateFileName)
-}
-
-// DataDirOrDefault is the daemon's persistent data directory: dataDir, or
-// DefaultDataDir when it is empty.
-func DataDirOrDefault(dataDir string) string {
 	if dataDir == "" {
-		return DefaultDataDir
+		dataDir = DefaultDataDir
 	}
-	return dataDir
+	return filepath.Join(dataDir, DefaultLocalStateDir, LocalStateFileName)
 }
 
 // MarshalLocalState produces the JSON wire form of vms with the lowest schema

spinifex/runtime/daemon/nats_classification_test.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
-	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/stretchr/testify/assert"
@@ -129,7 +128,6 @@ func TestEC2CommandNameCoversEveryCommand(t *testing.T) {
 	}{
 		{"AttachVolume", func(a *ec2v1.EC2CommandAttributes) { a.AttachVolume = true }},
 		{"DetachVolume", func(a *ec2v1.EC2CommandAttributes) { a.DetachVolume = true }},
-		{"DrainVolume", func(a *ec2v1.EC2CommandAttributes) { a.DrainVolume = true }},
 		{"AttachNetworkInterface", func(a *ec2v1.EC2CommandAttributes) { a.AttachENI = true }},
 		{"DetachNetworkInterface", func(a *ec2v1.EC2CommandAttributes) { a.DetachENI = true }},
 		{"AssociateIamInstanceProfile", func(a *ec2v1.EC2CommandAttributes) { a.AssociateIamInstanceProfile = true }},
@@ -187,28 +185,3 @@ func TestEC2CommandRecordsUnderTheCommandNotTheInstance(t *testing.T) {
 	assert.Empty(t, requestOutcomesFor(t, subject),
 		"the instance id must not reach the metric dimension")
 }
-
-// A drain runs off-thread so the command subscription is not held for the
-// length of the flush. Its point therefore has to be recorded by the goroutine
-// once the work finishes: recording at dispatch would call every drain an
-// instant success.
-func TestDrainVolumeRecordsItsPointAfterTheWork(t *testing.T) {
-	installOutcomeReader(t)
-	const instanceID, volumeID = "i-drain-metric", "vol-drain-metric"
-	daemon := drainTestDaemon(t, instanceID)
-	testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "OK\n")
-	subscribeEC2Commands(t, daemon, instanceID)
-
-	before := requestCountFor(t, ec2CmdAction("DrainVolume"), outcomeSuccess)
-	reply := drainRequest(t, daemon, instanceID, volumeID)
-
-	var ack ec2v1.DrainVolumeResponse
-	require.NoError(t, json.Unmarshal(reply.Data, &ack))
-	require.Equal(t, ec2v1.DrainVolumeStatusDrained, ack.Status)
-
-	// The goroutine records after it replies, so poll rather than assume the
-	// point has landed by the time the reply arrives.
-	require.Eventually(t, func() bool {
-		return requestCountFor(t, ec2CmdAction("DrainVolume"), outcomeSuccess) > before
-	}, 5*time.Second, 20*time.Millisecond, "the drain goroutine must record its own success point")
-}

spinifex/runtime/daemon/shutdown_test.go

@@ -13,6 +13,7 @@ import (
 
 	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	ebspolicy "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/policy"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/nats-io/nats.go"
@@ -676,36 +677,16 @@ func TestHandleShutdownDrain(t *testing.T) {
 	})
 }
 
-// stubEBSUnmount answers the daemon's ebs.<node>.unmount subject with a fixed
-// response, standing in for viperblockd. Returns the volumes it was asked to
-// unmount once the subscription has been drained.
-func stubEBSUnmount(t *testing.T, d *Daemon, resp viperblocklegacyv1.EBSUnMountResponse) func() []string {
+// stubEBSUnmount answers the daemon's unpublish subject, standing in for its
+// provider, with refusal when one is given. Returns the volumes it was asked
+// to unmount.
+func stubEBSUnmount(t *testing.T, d *Daemon, refusal *ebsprovider.ProviderError) func() []string {
 	t.Helper()
-	var (
-		mu      sync.Mutex
-		volumes []string
-	)
-	sub, err := d.natsConn.Subscribe("ebs."+d.node+".unmount", func(msg *nats.Msg) {
-		var req viperblocklegacyv1.EBSRequest
-		require.NoError(t, json.Unmarshal(msg.Data, &req))
-		mu.Lock()
-		volumes = append(volumes, req.Name)
-		mu.Unlock()
-		reply := resp
-		reply.Volume = req.Name
-		data, marshalErr := json.Marshal(reply)
-		require.NoError(t, marshalErr)
-		require.NoError(t, msg.Respond(data))
+	fake := serveFakeNodeProvider(t, d.natsConn, d.node)
+	fake.setUnpublish(func(string) ebsprovider.UnpublishVolumeResponse {
+		return ebsprovider.UnpublishVolumeResponse{Error: refusal}
 	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-	require.NoError(t, d.natsConn.Flush())
-
-	return func() []string {
-		mu.Lock()
-		defer mu.Unlock()
-		return append([]string(nil), volumes...)
-	}
+	return fake.unpublished
 }
 
 // drainRunningInstance is a guest in the state DRAIN finds it in: running, with
@@ -731,10 +712,8 @@ func TestHandleShutdownDrain_SealFailure(t *testing.T) {
 		require.NoError(t, daemon.jsManager.InitClusterStateBucket())
 		t.Cleanup(func() { _ = daemon.jsManager.DeleteShutdownMarker(daemon.node) })
 
-		unmounted := stubEBSUnmount(t, daemon, viperblocklegacyv1.EBSUnMountResponse{
-			Mounted: true,
-			Error:   "seal volume to predastore: transfer stalled: idle timeout",
-		})
+		unmounted := stubEBSUnmount(t, daemon, providerRefusal(ebsprovider.ErrorCodeInternal,
+			"seal volume to predastore: transfer stalled: idle timeout"))
 		daemon.vmMgr.Insert(drainRunningInstance("i-drain-seal-fail", "vol-drain-seal-fail"))
 
 		subject := "spinifex.cluster.shutdown.drain.sealfail"
@@ -770,7 +749,7 @@ func TestHandleShutdownDrain_SealFailure(t *testing.T) {
 		require.NoError(t, daemon.jsManager.InitClusterStateBucket())
 		t.Cleanup(func() { _ = daemon.jsManager.DeleteShutdownMarker(daemon.node) })
 
-		unmounted := stubEBSUnmount(t, daemon, viperblocklegacyv1.EBSUnMountResponse{})
+		unmounted := stubEBSUnmount(t, daemon, nil)
 		instance := drainRunningInstance("i-drain-seal-ok", "vol-drain-seal-ok")
 		daemon.vmMgr.Insert(instance)
 

spinifex/runtime/daemon/vm_adapters.go

@@ -19,6 +19,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
@@ -92,26 +93,129 @@ func (a *stateStoreAdapter) DeleteTerminatedInstance(id string) error {
 	return a.js.DeleteTerminatedInstance(id)
 }
 
-// volumeMounterAdapter satisfies vm.VolumeMounter by routing ebs.mount /
-// ebs.unmount NATS requests. Unmount also updates volume state via VolumeStateUpdater.
+// volumeMounterAdapter satisfies vm.VolumeMounter over the EBS contract's
+// PublishVolume and UnpublishVolume, sent at the cluster's one contract
+// version to this node's provider. Unmount also marks volumes available.
 type volumeMounterAdapter struct {
-	nc       *nats.Conn
 	node     string
 	volState vm.VolumeStateUpdater
+	provider mounterProvider
+	// abandonTimeout bounds each no-seal unpublish Abandon sends.
+	abandonTimeout time.Duration
 }
 
+// mounterProvider is the part of the EBS contract the mounter drives.
+type mounterProvider interface {
+	PublishVolume(context.Context, ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error)
+	UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error)
+}
+
+var _ mounterProvider = (*ebsprovider.NATSProvider)(nil)
+
 var _ vm.VolumeMounter = (*volumeMounterAdapter)(nil)
 
-// unmountSealTimeout bounds the ebs.unmount NATS request. The handler now drives
-// a synchronous block-map seal to predastore (S3 I/O), which runs AFTER
-// viperblockd's SIGKILL once its 120s KillProcess grace elapses — so this
-// must strictly exceed that grace plus a seal allowance, never merely match
-// it: an unmount that consumes the whole grace could otherwise never be
-// observed as successful.
-const unmountSealTimeout = 180 * time.Second
+func newVolumeMounterAdapter(provider mounterProvider, node string, volState vm.VolumeStateUpdater) *volumeMounterAdapter {
+	return &volumeMounterAdapter{
+		node:           node,
+		volState:       volState,
+		provider:       provider,
+		abandonTimeout: abandonTimeout,
+	}
+}
 
-func newVolumeMounterAdapter(nc *nats.Conn, node string, volState vm.VolumeStateUpdater) *volumeMounterAdapter {
-	return &volumeMounterAdapter{nc: nc, node: node, volState: volState}
+// publish mounts req's volume on this node through PublishVolume and records
+// the export on req. A failure worth retrying is vm.ErrMountRetryable; one
+// that leaves unknown whether the provider exported it is vm.ErrMountAmbiguous.
+func (a *volumeMounterAdapter) publish(ctx context.Context, accountID string, req *viperblocklegacyv1.EBSRequest) error {
+	// A caller already gone sends nothing, so there is no export to tear down;
+	// only a cancellation after the request went out leaves the outcome unknown.
+	if err := ctx.Err(); err != nil {
+		return fmt.Errorf("publish volume %s: %w", req.Name, err)
+	}
+	published, err := a.provider.PublishVolume(ebsprovider.WithAccountID(ctx, accountID), ebsprovider.PublishVolumeRequest{
+		Versioned: ebsprovider.NewLatestVersioned(),
+		VolumeID:  req.Name,
+		NodeID:    a.node,
+	})
+	if err != nil {
+		switch {
+		case publishRetryable(err):
+			return fmt.Errorf("publish volume %s: %w: %w", req.Name, err, vm.ErrMountRetryable)
+		case publishOutcomeUnknown(err):
+			return fmt.Errorf("publish volume %s: %w: %w", req.Name, err, vm.ErrMountAmbiguous)
+		}
+		return fmt.Errorf("publish volume %s: %w", req.Name, err)
+	}
+	applyExport(req, published.Export)
+	slog.InfoContext(ctx, "Published volume", "volume", req.Name, "node", a.node, "export_kind", string(published.Export.Kind))
+	return nil
+}
+
+// publishRetryable reports whether a failed publish is worth repeating later.
+// A connection that is down never sent the request, and not_found is how a
+// volume whose state is not loadable yet is reported; restore backs off on both.
+func publishRetryable(err error) bool {
+	if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrConnectionClosed) {
+		return true
+	}
+	var providerErr *ebsprovider.ProviderError
+	if !errors.As(err, &providerErr) {
+		return false
+	}
+	return providerErr.Code.Retryable() || providerErr.Code == ebsprovider.ErrorCodeNotFound
+}
+
+// publishOutcomeUnknown reports whether the provider may have exported the
+// volume despite the error: the request was sent but no usable reply came back,
+// or the caller stopped waiting for it. A connection found down sends nothing.
+func publishOutcomeUnknown(err error) bool {
+	for _, unknown := range []error{
+		context.DeadlineExceeded, context.Canceled, nats.ErrTimeout, ebsprovider.ErrUnreadableReply,
+		ebsprovider.ErrNoPublishedVolume, ebsprovider.ErrInvalidExport,
+	} {
+		if errors.Is(err, unknown) {
+			return true
+		}
+	}
+	return false
+}
+
+// applyExport records export on req as the one export field QEMU attaches.
+// The provider client has already validated it.
+func applyExport(req *viperblocklegacyv1.EBSRequest, export ebsprovider.VolumeExport) {
+	switch export.Kind {
+	case ebsprovider.ExportKindBlock:
+		req.NBDURI = ""
+		req.BlockDevice = export.Block.DevicePath
+		req.BlockSizeBytes = export.Block.SizeBytes
+	default:
+		req.NBDURI = export.NBD.NBDURI
+		req.BlockDevice = ""
+		req.BlockSizeBytes = 0
+	}
+}
+
+// unpublish tears down this node's export of req's volume through
+// UnpublishVolume, which seals first, and returns nil only once its block map
+// is sealed or the provider holds no export of it.
+func (a *volumeMounterAdapter) unpublish(ctx context.Context, accountID string, req viperblocklegacyv1.EBSRequest) error {
+	// The seal takes the provider's data-movement budget, not what is left of
+	// the caller's; the caller's trace is kept.
+	sealCtx := ebsprovider.WithAccountID(context.WithoutCancel(ctx), accountID)
+	_, err := a.provider.UnpublishVolume(sealCtx, ebsprovider.UnpublishVolumeRequest{
+		Versioned: ebsprovider.NewLatestVersioned(),
+		VolumeID:  req.Name,
+		NodeID:    a.node,
+	})
+	switch {
+	case err == nil:
+		return nil
+	case errors.Is(err, ebsprovider.ErrNotFound):
+		slog.InfoContext(ctx, "Unpublish found no volume, counting it as done", "volume", req.Name, "node", a.node)
+		return nil
+	default:
+		return fmt.Errorf("unpublish volume %s: %w", req.Name, err)
+	}
 }
 
 // ebsRequestWithTrace sends an ebs.* NATS request, opening a client span under
@@ -123,13 +227,10 @@ func newVolumeMounterAdapter(nc *nats.Conn, node string, volState vm.VolumeState
 // sweep or a recovery belongs to nobody, and crediting it to whoever happens
 // to be admin would be worse than leaving it blank.
 func ebsRequestWithTrace(ctx context.Context, nc *nats.Conn, accountID, subject string, data []byte, timeout time.Duration) (msg *nats.Msg, err error) {
-	attrs := []attribute.KeyValue{
+	attrs := append([]attribute.KeyValue{
 		attribute.String("messaging.system", "nats"),
 		attribute.String("messaging.destination.name", subject),
-	}
-	if accountID != "" {
-		attrs = append(attrs, attribute.String(natsmsg.AttrAccountID, accountID))
-	}
+	}, natsmsg.AccountAttrs(accountID)...)
 
 	ctx, span := otel.Tracer(daemonTracerName).Start(ctx, "NATS "+subject,
 		trace.WithSpanKind(trace.SpanKindClient),
@@ -162,7 +263,7 @@ func (a *volumeMounterAdapter) Mount(ctx context.Context, instance *vm.VM) error
 		var rbErrs []error
 		for _, idx := range mounted {
 			req := instance.EBSRequests.Requests[idx]
-			if err := a.unmountOne(ctx, instance.AccountID, req); err != nil {
+			if err := a.unpublish(ctx, instance.AccountID, req); err != nil {
 				slog.Error("Mount rollback: unmount failed",
 					"volume", req.Name, "err", err)
 				rbErrs = append(rbErrs, fmt.Errorf("unmount %s: %w", req.Name, err))
@@ -175,42 +276,14 @@ func (a *volumeMounterAdapter) Mount(ctx context.Context, instance *vm.VM) error
 	}
 
 	for k, v := range instance.EBSRequests.Requests {
-		ebsMountRequest, err := json.Marshal(v)
-		if err != nil {
-			slog.Error("Failed to marshal volume payload", "err", err)
-			return rollback(err)
-		}
-
-		reply, err := ebsRequestWithTrace(ctx, a.nc, instance.AccountID, viperblocklegacyv1.MountSubject(a.node), ebsMountRequest, 30*time.Second)
-
-		slog.Info("Mounting volume", "Vol", v.Name, "NBDURI", v.NBDURI)
-
-		if err != nil {
-			slog.Error("Failed to request EBS mount", "err", err)
-			// A missing responder means viperblockd has not finished starting;
-			// during recovery that is transient, so let the caller retry.
-			if errors.Is(err, nats.ErrNoResponders) {
-				return rollback(fmt.Errorf("ebs mount had no responder: %w", vm.ErrMountRetryable))
+		if err := a.publish(ctx, instance.AccountID, &instance.EBSRequests.Requests[k]); err != nil {
+			slog.Error("Failed to publish volume", "volume", v.Name, "err", err)
+			// The provider may have started this export, so it is torn down too.
+			if errors.Is(err, vm.ErrMountAmbiguous) {
+				mounted = append(mounted, k)
 			}
 			return rollback(err)
 		}
-
-		var ebsMountResponse viperblocklegacyv1.EBSMountResponse
-		if err := json.Unmarshal(reply.Data, &ebsMountResponse); err != nil {
-			slog.Error("Failed to unmarshal volume response:", "err", err)
-			return rollback(err)
-		}
-
-		if ebsMountResponse.Error != "" {
-			slog.Error("Failed to mount volume", "error", ebsMountResponse.Error, "retryable", ebsMountResponse.Retryable)
-			if ebsMountResponse.Retryable {
-				return rollback(fmt.Errorf("failed to mount volume: %s: %w", ebsMountResponse.Error, vm.ErrMountRetryable))
-			}
-			return rollback(fmt.Errorf("failed to mount volume: %s", ebsMountResponse.Error))
-		}
-
-		slog.Debug("Mounted volume successfully", "response", ebsMountResponse.URI)
-		instance.EBSRequests.Requests[k].NBDURI = ebsMountResponse.URI
 		mounted = append(mounted, k)
 	}
 
@@ -223,39 +296,22 @@ func (a *volumeMounterAdapter) Unmount(ctx context.Context, instance *vm.VM) err
 
 	var sealErrs []error
 	for _, ebsRequest := range instance.EBSRequests.Requests {
-		ebsUnMountRequest, err := json.Marshal(ebsRequest)
-		if err != nil {
-			slog.Error("Failed to marshal volume payload for unmount", "err", err)
-			sealErrs = append(sealErrs, fmt.Errorf("marshal unmount request for %s: %w", ebsRequest.Name, err))
+		// Every volume is attempted and the seal failures aggregated. An unsealed
+		// volume must NOT go available: a reattach on a node without the local
+		// WAL would find no checkpoint (bad superblock).
+		if err := a.unpublish(ctx, instance.AccountID, ebsRequest); err != nil {
+			slog.Error("Volume seal failed, leaving volume non-available",
+				"instance", instance.ID, "volume", ebsRequest.Name, "err", err)
+			sealErrs = append(sealErrs, fmt.Errorf("seal %s: %w", ebsRequest.Name, err))
 			continue
 		}
-
-		// ebs.unmount seals the block map to predastore. Every volume is
-		// attempted, and the failures are aggregated for the caller: the
-		// volume must NOT go available, because a reattach on a node without
-		// the local WAL would find no checkpoint (bad superblock).
-		sealed := true
-		msg, err := ebsRequestWithTrace(ctx, a.nc, instance.AccountID, viperblocklegacyv1.UnmountSubject(a.node), ebsUnMountRequest, unmountSealTimeout)
-		if err != nil {
-			slog.Error("Failed to unmount volume",
-				"name", ebsRequest.Name, "instance", instance.ID, "err", err)
-			sealErrs = append(sealErrs, fmt.Errorf("unmount %s: %w", ebsRequest.Name, err))
-			sealed = false
-		} else if sealErr := unmountResponseError(msg.Data); sealErr != nil {
-			slog.Error("Volume unmount seal failed, leaving volume non-available",
-				"instance", instance.ID, "volume", ebsRequest.Name, "err", sealErr)
-			sealErrs = append(sealErrs, fmt.Errorf("seal %s: %w", ebsRequest.Name, sealErr))
-			sealed = false
-		} else {
-			slog.Info("Unmounted volume",
-				"instance", instance.ID, "volume", ebsRequest.Name, "data", string(msg.Data))
-		}
+		slog.Info("Unmounted volume", "instance", instance.ID, "volume", ebsRequest.Name)
 
 		// Boot/root volumes must stay attached across stop/crash unmount: EFI is
 		// the wrong proxy for "boot" (BIOS-boot roots are Boot && !EFI), so gate on
 		// Boot too. Only DetachVolume and terminate release a boot volume; flipping
 		// it available here while the instance restarts splits the state record.
-		if sealed && !ebsRequest.EFI && !ebsRequest.Boot && a.volState != nil {
+		if !ebsRequest.EFI && !ebsRequest.Boot && a.volState != nil {
 			if err := a.volState.UpdateVolumeState(instance.AccountID, ebsRequest.Name, "available", "", ""); err != nil {
 				slog.Error("Failed to update volume state to available after unmount",
 					"volumeId", ebsRequest.Name, "err", err)
@@ -266,87 +322,71 @@ func (a *volumeMounterAdapter) Unmount(ctx context.Context, instance *vm.VM) err
 	return errors.Join(sealErrs...)
 }
 
-// abandonTimeout bounds one ebs.abandon request. Generous enough to cover
-// viperblockd's SIGKILL wait for an nbdkit stuck in the storage path, and far
-// short of the unmount's budget because nothing here seals.
+// abandonTimeout bounds one no-seal unpublish. Generous enough to cover a
+// provider killing an export stuck in the storage path, and far short of the
+// seal budget because nothing here seals.
 const abandonTimeout = 45 * time.Second
 
-// Abandon tears down this node's export of every volume the instance had open,
-// without sealing any of them.
+// Abandon tears down this node's export of every volume the instance had open
+// through UnpublishVolume with NoSeal, so none of them is sealed.
 //
 // Every volume is attempted and the failures are aggregated, because a volume
 // left exported is one the new owner cannot open — the caller is dropping a
 // superseded instance and needs to know whether it actually let go.
-func (a *volumeMounterAdapter) Abandon(ctx context.Context, instance *vm.VM, reason string) error {
+func (a *volumeMounterAdapter) Abandon(ctx context.Context, instance *vm.VM, reason string) (err error) {
+	ctx, span := otel.Tracer(daemonTracerName).Start(ctx, "ebs.mounter.abandon", trace.WithAttributes(append([]attribute.KeyValue{
+		attribute.String("instance.id", instance.ID),
+		attribute.String("abandon.reason", reason),
+	}, natsmsg.AccountAttrs(instance.AccountID)...)...))
+	defer func() { endOpSpan(span, err) }()
+
 	instance.EBSRequests.Mu.Lock()
 	defer instance.EBSRequests.Mu.Unlock()
 
 	var errs []error
 	for _, ebsRequest := range instance.EBSRequests.Requests {
-		payload, err := json.Marshal(viperblocklegacyv1.VolumeAbandonRequest{Volume: ebsRequest.Name, Reason: reason})
-		if err != nil {
-			errs = append(errs, fmt.Errorf("marshal abandon request for %s: %w", ebsRequest.Name, err))
-			continue
-		}
-
-		msg, err := ebsRequestWithTrace(ctx, a.nc, instance.AccountID,
-			viperblocklegacyv1.VolumeAbandonSubject(a.node), payload, abandonTimeout)
-		if err != nil {
-			errs = append(errs, fmt.Errorf("abandon %s: %w", ebsRequest.Name, err))
-			continue
-		}
-
-		var resp viperblocklegacyv1.VolumeAbandonResponse
-		if err := json.Unmarshal(msg.Data, &resp); err != nil {
-			errs = append(errs, fmt.Errorf("unmarshal abandon response for %s: %w", ebsRequest.Name, err))
-			continue
-		}
-		if resp.Error != "" {
-			errs = append(errs, fmt.Errorf("abandon %s: %s", ebsRequest.Name, resp.Error))
-			continue
-		}
-		if resp.Abandoned {
-			slog.WarnContext(ctx, "Gave up a volume export to the node that owns the instance now",
-				"instance", instance.ID, "volume", ebsRequest.Name, "reason", reason)
+		if err := a.abandonOne(ctx, instance, ebsRequest.Name, reason); err != nil {
+			errs = append(errs, err)
 		}
 	}
-
 	return errors.Join(errs...)
 }
 
-// MountOne sends ebs.mount for a single request and writes the resolved
-// NBDURI back into req.NBDURI. Used by hot-attach (Manager.AttachVolume).
-func (a *volumeMounterAdapter) MountOne(ctx context.Context, accountID string, req *viperblocklegacyv1.EBSRequest) error {
-	payload, err := json.Marshal(req)
-	if err != nil {
-		return fmt.Errorf("marshal ebs.mount request: %w", err)
+// abandonOne drops this node's export of volumeID unsealed. A volume the
+// provider does not know is one this node no longer holds.
+func (a *volumeMounterAdapter) abandonOne(ctx context.Context, instance *vm.VM, volumeID, reason string) error {
+	abandonCtx, cancel := context.WithTimeout(ebsprovider.WithAccountID(ctx, instance.AccountID), a.abandonTimeout)
+	defer cancel()
+	unpublished, err := a.provider.UnpublishVolume(abandonCtx, ebsprovider.UnpublishVolumeRequest{
+		Versioned: ebsprovider.NewLatestVersioned(),
+		VolumeID:  volumeID,
+		NodeID:    a.node,
+		NoSeal:    true,
+	})
+	switch {
+	case errors.Is(err, ebsprovider.ErrNotFound):
+		return nil
+	case err != nil:
+		return fmt.Errorf("abandon %s (%s): %w", volumeID, reason, err)
 	}
-
-	reply, err := ebsRequestWithTrace(ctx, a.nc, accountID, viperblocklegacyv1.MountSubject(a.node), payload, 30*time.Second)
-	if err != nil {
-		return fmt.Errorf("ebs.mount NATS request: %w", err)
+	if unpublished.TornDown {
+		slog.WarnContext(ctx, "Gave up a volume export to the node that owns the instance now",
+			"instance", instance.ID, "volume", volumeID, "reason", reason)
 	}
-
-	var resp viperblocklegacyv1.EBSMountResponse
-	if err := json.Unmarshal(reply.Data, &resp); err != nil {
-		return fmt.Errorf("unmarshal ebs.mount response: %w", err)
-	}
-	if resp.Error != "" {
-		return fmt.Errorf("ebs.mount returned error: %s", resp.Error)
-	}
-	if resp.URI == "" {
-		return vm.ErrMountAmbiguous
-	}
-
-	req.NBDURI = resp.URI
 	return nil
 }
 
-// UnmountOne sends ebs.unmount and returns any error. The handler seals the
-// volume's block map to predastore, so the caller decides whether a failure
-// blocks the volume's available transition.
+// MountOne mounts a single request and writes the resolved export back into
+// req. Used by hot-attach (Manager.AttachVolume).
+func (a *volumeMounterAdapter) MountOne(ctx context.Context, accountID string, req *viperblocklegacyv1.EBSRequest) error {
+	return a.publish(ctx, accountID, req)
+}
+
+// UnmountOne unmounts a single request and returns any error. The unmount
+// seals the volume's block map to the backend, so the caller decides whether
+// a failure blocks the volume's available transition.
 func (a *volumeMounterAdapter) UnmountOne(ctx context.Context, accountID string, req viperblocklegacyv1.EBSRequest) error {
-	if err := a.unmountOne(ctx, accountID, req); err != nil {
+	if err := a.unpublish(ctx, accountID, req); err != nil {
 		slog.Error("UnmountOne failed", "volume", req.Name, "err", err)
 		return err
 	}
@@ -354,42 +394,6 @@ func (a *volumeMounterAdapter) UnmountOne(ctx context.Context, accountID string,
 	return nil
 }
 
-// unmountOne sends ebs.unmount and returns any error.
-func (a *volumeMounterAdapter) unmountOne(ctx context.Context, accountID string, req viperblocklegacyv1.EBSRequest) error {
-	payload, err := json.Marshal(req)
-	if err != nil {
-		return fmt.Errorf("marshal unmount request: %w", err)
-	}
-	msg, err := ebsRequestWithTrace(ctx, a.nc, accountID, viperblocklegacyv1.UnmountSubject(a.node), payload, unmountSealTimeout)
-	if err != nil {
-		return fmt.Errorf("ebs.unmount NATS request: %w", err)
-	}
-	return unmountResponseError(msg.Data)
-}
-
-// unmountResponseError reports a seal/unmount failure from an ebs.unmount
-// response payload: a non-empty Error, or a volume still reported mounted.
-// Returns nil when the unmount and its block-map seal succeeded, or when the
-// volume was already unmounted and sealed (NotFound) — viperblockd only
-// reports NotFound once the seal itself has completed, so a retry landing
-// here is an idempotent success, not a failure.
-func unmountResponseError(data []byte) error {
-	var resp viperblocklegacyv1.EBSUnMountResponse
-	if err := json.Unmarshal(data, &resp); err != nil {
-		return fmt.Errorf("unmarshal unmount response: %w", err)
-	}
-	if resp.NotFound {
-		return nil
-	}
-	if resp.Error != "" {
-		return fmt.Errorf("ebs.unmount returned error: %s", resp.Error)
-	}
-	if resp.Mounted {
-		return fmt.Errorf("volume still mounted after unmount")
-	}
-	return nil
-}
-
 // instanceTypeResolverAdapter satisfies vm.InstanceTypeResolver by projecting
 // EC2 instance type info into the SDK-agnostic vm.InstanceTypeSpec.
 type instanceTypeResolverAdapter struct {
@@ -673,7 +677,7 @@ func (d *Daemon) buildVMManagerDeps() vm.Deps {
 	return vm.Deps{
 		NodeID:             d.node,
 		StateStore:         d.stateStore,
-		VolumeMounter:      newVolumeMounterAdapter(d.natsConn, d.node, d.volumeService),
+		VolumeMounter:      newVolumeMounterAdapter(d.ebsProvider, d.node, d.volumeService),
 		NetworkPlumber:     d.networkPlumber,
 		InstanceTypes:      newInstanceTypeResolverAdapter(d.resourceMgr),
 		Resources:          newResourceControllerAdapter(d.resourceMgr),
@@ -696,11 +700,6 @@ func (d *Daemon) buildVMManagerDeps() vm.Deps {
 		DeviceDeletedTimeout:       d.deviceDeletedTimeout,
 		ConsumeCleanShutdownMarker: d.consumeCleanShutdownMarker(),
 		BackingStoreReady:          func() bool { return d.checkPredastoreReady() && d.checkViperblockReady() },
-		UEFIVarStore: vm.UEFIVarStoreDeps{
-			Objects: d.instanceObjects,
-			Bucket:  d.config.Predastore.Bucket,
-			DataDir: DataDirOrDefault(d.config.DataDir),
-		},
 	}
 }
 

spinifex/runtime/daemon/vm_adapters_contract_test.go

@@ -0,0 +1,749 @@
+package daemon
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs/natsserve"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+var mounterTestNode atomic.Int64
+
+// newMounterTestNode returns a node name no other test on the shared NATS
+// server subscribes for, so per-node subjects never cross tests.
+func newMounterTestNode() string {
+	return fmt.Sprintf("node-mounter-%d", mounterTestNode.Add(1))
+}
+
+// newNATSMounter returns a mounter driving node's provider over nc.
+func newNATSMounter(nc *nats.Conn, node string, volState vm.VolumeStateUpdater) *volumeMounterAdapter {
+	return newVolumeMounterAdapter(ebsprovider.NewNATSProvider(nc, ebsProviderRequestTimeout), node, volState)
+}
+
+func connectMounterTest(t *testing.T) *nats.Conn {
+	t.Helper()
+	nc, err := nats.Connect(sharedNATSURL)
+	require.NoError(t, err)
+	t.Cleanup(nc.Close)
+	return nc
+}
+
+// replyJSON answers every request on subject with reply(request body).
+func replyJSON(t *testing.T, nc *nats.Conn, subject string, reply func(data []byte) any) {
+	t.Helper()
+	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
+		data, err := json.Marshal(reply(msg.Data))
+		if err != nil {
+			panic(err)
+		}
+		_ = msg.Respond(data)
+	})
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+}
+
+// newMemoryProvider returns a memory provider holding volumeIDs.
+func newMemoryProvider(t *testing.T, volumeIDs ...string) *ebsprovider.MemoryProvider {
+	t.Helper()
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
+	for _, id := range volumeIDs {
+		_, err := provider.CreateVolume(t.Context(), ebsprovider.CreateVolumeRequest{
+			Versioned:     ebsprovider.NewVersioned(),
+			VolumeID:      id,
+			CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 20},
+		})
+		require.NoError(t, err)
+	}
+	return provider
+}
+
+// serveMemoryProvider serves provider as nodeID's local provider.
+func serveMemoryProvider(t *testing.T, nc *nats.Conn, provider *ebsprovider.MemoryProvider, nodeID string) {
+	t.Helper()
+	stop, err := natsserve.Serve(t.Context(), nc, provider, natsserve.Options{NoQueueGroup: true, NodeID: nodeID})
+	require.NoError(t, err)
+	t.Cleanup(stop)
+}
+
+func volumeState(t *testing.T, provider *ebsprovider.MemoryProvider, volumeID string) ebsprovider.VolumeState {
+	t.Helper()
+	volume, err := provider.GetVolume(t.Context(), ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID})
+	require.NoError(t, err)
+	return volume.State
+}
+
+func mounterTestVM(volumeIDs ...string) *vm.VM {
+	instance := &vm.VM{ID: "i-mounter", AccountID: testAccountID}
+	for _, id := range volumeIDs {
+		instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{Name: id})
+	}
+	return instance
+}
+
+// contractCall is one request a fakeNodeProvider or a spy received.
+type contractCall struct {
+	subject string
+	version uint16
+	volume  string
+	noSeal  bool
+	account string
+}
+
+// fakeNodeProvider answers node's publish and unpublish subjects with the
+// replies the test sets, echoing each request's version unless a publish reply
+// sets its own, and records them.
+type fakeNodeProvider struct {
+	mu        sync.Mutex
+	publish   func(volumeID string) ebsprovider.PublishVolumeResponse
+	unpublish func(volumeID string) ebsprovider.UnpublishVolumeResponse
+	calls     []contractCall
+}
+
+// serveFakeNodeProvider starts a fake that publishes every volume as an NBD
+// export and unpublishes it successfully until the test says otherwise.
+func serveFakeNodeProvider(t *testing.T, nc *nats.Conn, node string) *fakeNodeProvider {
+	t.Helper()
+	f := &fakeNodeProvider{
+		publish:   func(volumeID string) ebsprovider.PublishVolumeResponse { return publishedNBD("nbd://" + volumeID) },
+		unpublish: func(string) ebsprovider.UnpublishVolumeResponse { return ebsprovider.UnpublishVolumeResponse{} },
+	}
+	publishSubject, err := ebsprovider.PublishSubject(node)
+	require.NoError(t, err)
+	replyJSON(t, nc, publishSubject, func(data []byte) any {
+		var req ebsprovider.PublishVolumeRequest
+		_ = json.Unmarshal(data, &req)
+		f.mu.Lock()
+		f.calls = append(f.calls, contractCall{subject: publishSubject, version: req.SchemaVersion, volume: req.VolumeID})
+		reply := f.publish
+		f.mu.Unlock()
+		resp := reply(req.VolumeID)
+		if resp.SchemaVersion == 0 {
+			resp.Versioned = ebsprovider.ReplyVersioned(req.SchemaVersion)
+		}
+		if resp.Published != nil {
+			published := *resp.Published
+			published.VolumeID, published.NodeID = req.VolumeID, req.NodeID
+			resp.Published = &published
+		}
+		return resp
+	})
+	unpublishSubject, err := ebsprovider.UnpublishSubject(node)
+	require.NoError(t, err)
+	replyJSON(t, nc, unpublishSubject, func(data []byte) any {
+		var req ebsprovider.UnpublishVolumeRequest
+		_ = json.Unmarshal(data, &req)
+		f.mu.Lock()
+		f.calls = append(f.calls, contractCall{subject: unpublishSubject, version: req.SchemaVersion, volume: req.VolumeID, noSeal: req.NoSeal})
+		reply := f.unpublish
+		f.mu.Unlock()
+		resp := reply(req.VolumeID)
+		resp.Versioned = ebsprovider.ReplyVersioned(req.SchemaVersion)
+		return resp
+	})
+	return f
+}
+
+func (f *fakeNodeProvider) setPublish(reply func(volumeID string) ebsprovider.PublishVolumeResponse) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.publish = reply
+}
+
+func (f *fakeNodeProvider) setUnpublish(reply func(volumeID string) ebsprovider.UnpublishVolumeResponse) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.unpublish = reply
+}
+
+func (f *fakeNodeProvider) snapshot() []contractCall {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]contractCall(nil), f.calls...)
+}
+
+// unpublished returns, in order, the volumes the fake was asked to unpublish.
+func (f *fakeNodeProvider) unpublished() []string {
+	var volumes []string
+	for _, call := range f.snapshot() {
+		if strings.HasSuffix(call.subject, ".unmount") {
+			volumes = append(volumes, call.volume)
+		}
+	}
+	return volumes
+}
+
+func publishedNBD(uri string) ebsprovider.PublishVolumeResponse {
+	return ebsprovider.PublishVolumeResponse{Published: &ebsprovider.PublishedVolume{Export: ebsprovider.NewNBDExport(uri)}}
+}
+
+func providerRefusal(code ebsprovider.ErrorCode, message string) *ebsprovider.ProviderError {
+	return &ebsprovider.ProviderError{Code: code, Message: message}
+}
+
+func publishRefusal(code ebsprovider.ErrorCode) func(string) ebsprovider.PublishVolumeResponse {
+	return func(string) ebsprovider.PublishVolumeResponse {
+		return ebsprovider.PublishVolumeResponse{Error: providerRefusal(code, "refused: "+string(code))}
+	}
+}
+
+func unpublishRefusal(code ebsprovider.ErrorCode, message string) ebsprovider.UnpublishVolumeResponse {
+	return ebsprovider.UnpublishVolumeResponse{Error: providerRefusal(code, message)}
+}
+
+// spyOn records every request sent on subject without answering it, so a
+// request the mounter sends there is seen even when another subscriber
+// answers it.
+func spyOn(t *testing.T, nc *nats.Conn, subject string) func() []contractCall {
+	t.Helper()
+	var (
+		mu    sync.Mutex
+		calls []contractCall
+	)
+	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
+		var req ebsprovider.UnpublishVolumeRequest
+		_ = json.Unmarshal(msg.Data, &req)
+		mu.Lock()
+		calls = append(calls, contractCall{
+			subject: msg.Subject, version: req.SchemaVersion, volume: req.VolumeID,
+			noSeal: req.NoSeal, account: natsmsg.AccountIDFromMsg(msg),
+		})
+		mu.Unlock()
+	})
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	return func() []contractCall {
+		mu.Lock()
+		defer mu.Unlock()
+		return append([]contractCall(nil), calls...)
+	}
+}
+
+// Every mount, unmount and abandon goes to the node's provider over the
+// contract at the cluster's one version, and none uses a legacy subject.
+func TestVolumeMounterAdapter_EveryRequestAtVersion3OverTheContract(t *testing.T) {
+	nc := connectMounterTest(t)
+	node := newMounterTestNode()
+	provider := newMemoryProvider(t, "vol-a", "vol-b", "vol-hot", "vol-moved")
+	serveMemoryProvider(t, nc, provider, node)
+	contractCalls := spyOn(t, nc, "ebs.provider.v1."+node+".*")
+	var legacySpies []func() []contractCall
+	for _, subject := range []string{"ebs." + node + ".*", "ebs.mount", "ebs.unmount", "ebs.abandon"} {
+		legacySpies = append(legacySpies, spyOn(t, nc, subject))
+	}
+	volState := &recordingVolState{}
+	adapter := newNATSMounter(nc, node, volState)
+
+	instance := mounterTestVM("vol-a", "vol-b")
+	require.NoError(t, adapter.Mount(t.Context(), instance))
+	for _, req := range instance.EBSRequests.Requests {
+		assert.Equal(t, "nbd+unix:///?socket=/memory/"+req.Name+".sock", req.NBDURI, "the published export must reach the request")
+		assert.Equal(t, ebsprovider.VolumeStateInUse, volumeState(t, provider, req.Name))
+	}
+
+	hot := &viperblocklegacyv1.EBSRequest{Name: "vol-hot"}
+	require.NoError(t, adapter.MountOne(t.Context(), testAccountID, hot))
+	assert.Equal(t, "nbd+unix:///?socket=/memory/vol-hot.sock", hot.NBDURI)
+	require.NoError(t, adapter.UnmountOne(t.Context(), testAccountID, *hot))
+	assert.Equal(t, ebsprovider.VolumeStateAvailable, volumeState(t, provider, "vol-hot"))
+
+	require.NoError(t, adapter.Unmount(t.Context(), instance))
+	for _, id := range []string{"vol-a", "vol-b"} {
+		assert.Equal(t, ebsprovider.VolumeStateAvailable, volumeState(t, provider, id), "unmount must unpublish %s", id)
+	}
+	assert.ElementsMatch(t, []string{"vol-a:available", "vol-b:available"}, volState.snapshot())
+
+	moved := mounterTestVM("vol-moved")
+	require.NoError(t, adapter.Mount(t.Context(), moved))
+	require.NoError(t, adapter.Abandon(t.Context(), moved, "superseded"))
+	assert.Equal(t, ebsprovider.VolumeStateAvailable, volumeState(t, provider, "vol-moved"), "abandon must give the export up")
+
+	calls := contractCalls()
+	assert.Len(t, calls, 8, "two mounts, a hot attach and detach, two unmounts, a mount and an abandon")
+	for _, call := range calls {
+		assert.Equal(t, ebsprovider.SchemaVersion, call.version, "%s for %s", call.subject, call.volume)
+	}
+	for _, legacyCalls := range legacySpies {
+		assert.Empty(t, legacyCalls(), "no request may use a legacy subject")
+	}
+}
+
+// A relaunch refused as retry-later, or because the volume state is not found
+// yet, backs off as restore expects and mounts once the provider is ready.
+func TestVolumeMounterAdapter_RetryLaterRefusalBacksOffAndRecovers(t *testing.T) {
+	for _, code := range []ebsprovider.ErrorCode{ebsprovider.ErrorCodeUnavailable, ebsprovider.ErrorCodeNotFound} {
+		t.Run(string(code), func(t *testing.T) {
+			nc := connectMounterTest(t)
+			node := newMounterTestNode()
+			fake := serveFakeNodeProvider(t, nc, node)
+			fake.setPublish(publishRefusal(code))
+			adapter := newNATSMounter(nc, node, nil)
+
+			instance := mounterTestVM("vol-relaunch")
+			err := adapter.Mount(t.Context(), instance)
+			require.ErrorIs(t, err, vm.ErrMountRetryable, "restore must back off rather than give up")
+			hot := &viperblocklegacyv1.EBSRequest{Name: "vol-hot"}
+			require.ErrorIs(t, adapter.MountOne(t.Context(), testAccountID, hot), vm.ErrMountRetryable)
+
+			fake.setPublish(func(string) ebsprovider.PublishVolumeResponse { return publishedNBD("nbd://ready") })
+			require.NoError(t, adapter.Mount(t.Context(), instance))
+			assert.Equal(t, "nbd://ready", instance.EBSRequests.Requests[0].NBDURI)
+		})
+	}
+}
+
+// A refusal that retrying cannot fix fails the mount outright, so restore
+// does not back off forever against a provider that will never accept it.
+func TestVolumeMounterAdapter_PermanentRefusalIsNotRetryable(t *testing.T) {
+	codes := []ebsprovider.ErrorCode{
+		ebsprovider.ErrorCodeVolumeInUse, ebsprovider.ErrorCodeVolumeHeldElsewhere,
+		ebsprovider.ErrorCodeInternal, ebsprovider.ErrorCodeInvalidArgument,
+	}
+	for _, code := range codes {
+		t.Run(string(code), func(t *testing.T) {
+			nc := connectMounterTest(t)
+			node := newMounterTestNode()
+			fake := serveFakeNodeProvider(t, nc, node)
+			fake.setPublish(publishRefusal(code))
+			adapter := newNATSMounter(nc, node, nil)
+
+			err := adapter.Mount(t.Context(), mounterTestVM("vol-refused"))
+			require.Error(t, err)
+			assert.NotErrorIs(t, err, vm.ErrMountRetryable)
+		})
+	}
+}
+
+// A volume another node holds is refused with the held-elsewhere code, which
+// a provider only sends at version 3, and the refusal reaches Mount's caller
+// as the sentinel that tells it apart from an ordinary volume in use.
+func TestVolumeMounterAdapter_HeldElsewhereReachesTheMountCaller(t *testing.T) {
+	nc := connectMounterTest(t)
+	owner, other := newMounterTestNode(), newMounterTestNode()
+	provider := newMemoryProvider(t, "vol-held")
+	serveMemoryProvider(t, nc, provider, owner)
+	serveMemoryProvider(t, nc, provider, other)
+
+	require.NoError(t, newNATSMounter(nc, owner, nil).Mount(t.Context(), mounterTestVM("vol-held")))
+
+	err := newNATSMounter(nc, other, nil).Mount(t.Context(), mounterTestVM("vol-held"))
+	require.ErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere)
+	var providerErr *ebsprovider.ProviderError
+	require.ErrorAs(t, err, &providerErr)
+	assert.Equal(t, ebsprovider.ErrorCodeVolumeHeldElsewhere, providerErr.Code)
+	assert.Equal(t, ebsprovider.VolumeStateInUse, volumeState(t, provider, "vol-held"), "the refused mount must not take the volume from its owner")
+}
+
+// The unpublish budget is the provider's data-movement budget, not the
+// request timeout every other call gets: a seal that outlasts the request
+// timeout still completes, where a publish that slow fails.
+func TestVolumeMounterAdapter_UnpublishOutlastingRequestTimeoutCompletes(t *testing.T) {
+	const requestTimeout = 100 * time.Millisecond
+	const sealTime = 4 * requestTimeout
+
+	nc := connectMounterTest(t)
+	node := newMounterTestNode()
+	fake := serveFakeNodeProvider(t, nc, node)
+	fake.setUnpublish(func(string) ebsprovider.UnpublishVolumeResponse {
+		time.Sleep(sealTime)
+		return ebsprovider.UnpublishVolumeResponse{}
+	})
+	fake.setPublish(func(volumeID string) ebsprovider.PublishVolumeResponse {
+		time.Sleep(sealTime)
+		return publishedNBD("nbd://" + volumeID)
+	})
+
+	volState := &recordingVolState{}
+	adapter := newNATSMounter(nc, node, volState)
+	adapter.provider = ebsprovider.NewNATSProvider(nc, requestTimeout)
+
+	require.NoError(t, adapter.Unmount(t.Context(), mounterTestVM("vol-slow-seal")))
+	assert.Equal(t, []string{"vol-slow-seal:available"}, volState.snapshot())
+	require.NoError(t, adapter.UnmountOne(t.Context(), testAccountID, viperblocklegacyv1.EBSRequest{Name: "vol-slow-seal"}))
+
+	err := adapter.MountOne(t.Context(), testAccountID, &viperblocklegacyv1.EBSRequest{Name: "vol-slow-seal"})
+	require.ErrorIs(t, err, context.DeadlineExceeded, "the publish must be held to the request timeout")
+}
+
+// A failed volume rolls back the volumes this Mount already published, and a
+// missing volume backs off rather than failing the relaunch.
+func TestVolumeMounterAdapter_MountRollsBackOnFailure(t *testing.T) {
+	nc := connectMounterTest(t)
+	node := newMounterTestNode()
+	provider := newMemoryProvider(t, "vol-present")
+	serveMemoryProvider(t, nc, provider, node)
+	adapter := newNATSMounter(nc, node, nil)
+
+	err := adapter.Mount(t.Context(), mounterTestVM("vol-present", "vol-missing"))
+	require.ErrorIs(t, err, vm.ErrMountRetryable)
+	assert.ErrorIs(t, err, ebsprovider.ErrNotFound)
+	assert.Equal(t, ebsprovider.VolumeStateAvailable, volumeState(t, provider, "vol-present"), "the published volume must be unpublished")
+}
+
+// stubMounterProvider publishes a fixed export without a NATS round trip.
+type stubMounterProvider struct {
+	export ebsprovider.VolumeExport
+}
+
+var _ mounterProvider = (*stubMounterProvider)(nil)
+
+func (s *stubMounterProvider) PublishVolume(_ context.Context, req ebsprovider.PublishVolumeRequest) (*ebsprovider.PublishedVolume, error) {
+	return &ebsprovider.PublishedVolume{VolumeID: req.VolumeID, NodeID: req.NodeID, Export: s.export}, nil
+}
+
+func (s *stubMounterProvider) UnpublishVolume(context.Context, ebsprovider.UnpublishVolumeRequest) (*ebsprovider.UnpublishedVolume, error) {
+	return &ebsprovider.UnpublishedVolume{}, nil
+}
+
+// A block export replaces whatever export the request carried before, so a
+// relaunch never leaves both an NBD URI and a block device set.
+func TestVolumeMounterAdapter_BlockExportReplacesNBDURI(t *testing.T) {
+	adapter := newVolumeMounterAdapter(&stubMounterProvider{export: ebsprovider.VolumeExport{
+		Kind:  ebsprovider.ExportKindBlock,
+		Block: &ebsprovider.BlockExport{DevicePath: "/dev/sdx", SizeBytes: 1 << 30},
+	}}, "node-stub", nil)
+
+	req := &viperblocklegacyv1.EBSRequest{Name: "vol-block", NBDURI: "nbd://stale"}
+	require.NoError(t, adapter.MountOne(t.Context(), testAccountID, req))
+	assert.Empty(t, req.NBDURI)
+	assert.Equal(t, "/dev/sdx", req.BlockDevice)
+	assert.Equal(t, uint64(1<<30), req.BlockSizeBytes)
+
+	adapter.provider = &stubMounterProvider{export: ebsprovider.NewNBDExport("nbd://fresh")}
+	require.NoError(t, adapter.MountOne(t.Context(), testAccountID, req))
+	assert.Equal(t, "nbd://fresh", req.NBDURI)
+	assert.Empty(t, req.BlockDevice)
+	assert.Zero(t, req.BlockSizeBytes)
+}
+
+// A publish whose outcome is unknown may still have started an export: the
+// reply carried no usable export, could not be read, or never came. Hot attach
+// gets ErrMountAmbiguous with the request untouched, and Mount tears it down.
+func TestVolumeMounterAdapter_AmbiguousPublishIsTornDown(t *testing.T) {
+	const requestTimeout = 100 * time.Millisecond
+	replies := map[string]struct {
+		reply   func() ebsprovider.PublishVolumeResponse
+		timeout time.Duration
+		cause   error
+	}{
+		"NoPublishedVolume": {
+			reply: func() ebsprovider.PublishVolumeResponse { return ebsprovider.PublishVolumeResponse{} },
+			cause: ebsprovider.ErrNoPublishedVolume,
+		},
+		"EmptyExport": {
+			reply: func() ebsprovider.PublishVolumeResponse { return publishedNBD("") },
+			cause: ebsprovider.ErrInvalidExport,
+		},
+		"UnsupportedReplyVersion": {
+			reply: func() ebsprovider.PublishVolumeResponse {
+				resp := publishedNBD("nbd://future")
+				resp.SchemaVersion = ebsprovider.SchemaVersion + 1
+				return resp
+			},
+			cause: ebsprovider.ErrUnsupportedVersion,
+		},
+		"Timeout": {
+			reply: func() ebsprovider.PublishVolumeResponse {
+				time.Sleep(4 * requestTimeout)
+				return publishedNBD("nbd://late")
+			},
+			timeout: requestTimeout,
+			cause:   context.DeadlineExceeded,
+		},
+	}
+	for name, tt := range replies {
+		t.Run(name, func(t *testing.T) {
+			nc := connectMounterTest(t)
+			node := newMounterTestNode()
+			fake := serveFakeNodeProvider(t, nc, node)
+			fake.setPublish(func(string) ebsprovider.PublishVolumeResponse { return tt.reply() })
+			adapter := newNATSMounter(nc, node, nil)
+			if tt.timeout > 0 {
+				adapter.provider = ebsprovider.NewNATSProvider(nc, tt.timeout)
+			}
+
+			hot := &viperblocklegacyv1.EBSRequest{Name: "vol-hot", NBDURI: "nbd://stale"}
+			err := adapter.MountOne(t.Context(), testAccountID, hot)
+			require.ErrorIs(t, err, vm.ErrMountAmbiguous)
+			require.ErrorIs(t, err, tt.cause)
+			assert.NotErrorIs(t, err, vm.ErrMountRetryable)
+			assert.Equal(t, "nbd://stale", hot.NBDURI, "a failed publish must not overwrite the request's export")
+
+			err = adapter.Mount(t.Context(), mounterTestVM("vol-ambiguous"))
+			require.ErrorIs(t, err, vm.ErrMountAmbiguous)
+			assert.Equal(t, []string{"vol-ambiguous"}, fake.unpublished(), "the ambiguous volume must be unpublished")
+		})
+	}
+}
+
+// A caller that gives up during a later volume's publish still gets that volume
+// and the ones already published unpublished, rather than left exported.
+func TestVolumeMounterAdapter_MountRollbackOutlivesCancelledCaller(t *testing.T) {
+	nc := connectMounterTest(t)
+	node := newMounterTestNode()
+	fake := serveFakeNodeProvider(t, nc, node)
+	arrived, release := make(chan struct{}), make(chan struct{})
+	t.Cleanup(func() { close(release) })
+	fake.setPublish(func(volumeID string) ebsprovider.PublishVolumeResponse {
+		if volumeID == "vol-slow" {
+			close(arrived)
+			<-release
+		}
+		return publishedNBD("nbd://" + volumeID)
+	})
+	adapter := newNATSMounter(nc, node, nil)
+
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	go func() {
+		<-arrived
+		cancel()
+	}()
+	err := adapter.Mount(ctx, mounterTestVM("vol-first", "vol-slow"))
+	require.ErrorIs(t, err, context.Canceled)
+	require.ErrorIs(t, err, vm.ErrMountAmbiguous)
+	assert.NotErrorIs(t, err, vm.ErrMountRetryable)
+	assert.ElementsMatch(t, []string{"vol-first", "vol-slow"}, fake.unpublished(),
+		"the published volume and the one whose publish the provider received must be unpublished despite the cancelled caller")
+}
+
+// A hot attach whose caller gives up after the provider received the publish
+// is ambiguous, so the attach unpublishes it, and that teardown still reaches
+// the provider under the cancelled context.
+func TestVolumeMounterAdapter_MountOneCancelledAfterSendIsAmbiguous(t *testing.T) {
+	nc := connectMounterTest(t)
+	node := newMounterTestNode()
+	fake := serveFakeNodeProvider(t, nc, node)
+	arrived, release := make(chan struct{}), make(chan struct{})
+	t.Cleanup(func() { close(release) })
+	fake.setPublish(func(volumeID string) ebsprovider.PublishVolumeResponse {
+		close(arrived)
+		<-release
+		return publishedNBD("nbd://" + volumeID)
+	})
+	adapter := newNATSMounter(nc, node, nil)
+
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	go func() {
+		<-arrived
+		cancel()
+	}()
+	req := &viperblocklegacyv1.EBSRequest{Name: "vol-hot", NBDURI: "nbd://stale"}
+	err := adapter.MountOne(ctx, testAccountID, req)
+	require.ErrorIs(t, err, context.Canceled)
+	require.ErrorIs(t, err, vm.ErrMountAmbiguous)
+	assert.NotErrorIs(t, err, vm.ErrMountRetryable)
+	assert.Equal(t, "nbd://stale", req.NBDURI, "a failed publish must not overwrite the request's export")
+
+	require.NoError(t, adapter.UnmountOne(ctx, testAccountID, *req))
+	assert.Equal(t, []string{"vol-hot"}, fake.unpublished())
+}
+
+// A caller already gone when a publish would start sends nothing, so neither
+// call is ambiguous and Mount unpublishes only the volumes it published.
+func TestVolumeMounterAdapter_PublishAfterCallerGoneSendsNothing(t *testing.T) {
+	nc := connectMounterTest(t)
+	node := newMounterTestNode()
+	fake := serveFakeNodeProvider(t, nc, node)
+	adapter := newNATSMounter(nc, node, nil)
+
+	cancelled, cancel := context.WithCancel(t.Context())
+	cancel()
+	err := adapter.MountOne(cancelled, testAccountID, &viperblocklegacyv1.EBSRequest{Name: "vol-hot"})
+	require.ErrorIs(t, err, context.Canceled)
+	assert.NotErrorIs(t, err, vm.ErrMountAmbiguous)
+
+	err = adapter.Mount(cancelled, mounterTestVM("vol-never"))
+	require.ErrorIs(t, err, context.Canceled)
+	assert.NotErrorIs(t, err, vm.ErrMountAmbiguous)
+	assert.Empty(t, fake.snapshot(), "no request may reach the provider for a caller already gone")
+}
+
+// Every contract request names the account it acts for on the wire, so the
+// provider's server span is attributable, and an unattributed one names none.
+func TestVolumeMounterAdapter_ContractRequestsCarryTheAccount(t *testing.T) {
+	nc := connectMounterTest(t)
+	node := newMounterTestNode()
+	serveFakeNodeProvider(t, nc, node)
+	calls := spyOn(t, nc, "ebs.provider.v1."+node+".*")
+	adapter := newNATSMounter(nc, node, nil)
+
+	instance := mounterTestVM("vol-a")
+	require.NoError(t, adapter.Mount(t.Context(), instance))
+	require.NoError(t, adapter.Unmount(t.Context(), instance))
+	require.NoError(t, adapter.Abandon(t.Context(), instance, "superseded"))
+	hot := &viperblocklegacyv1.EBSRequest{Name: "vol-hot"}
+	require.NoError(t, adapter.MountOne(t.Context(), testAccountID, hot))
+	require.NoError(t, adapter.UnmountOne(t.Context(), testAccountID, *hot))
+	require.NoError(t, adapter.MountOne(t.Context(), "", &viperblocklegacyv1.EBSRequest{Name: "vol-none"}))
+
+	require.Eventually(t, func() bool { return len(calls()) == 6 }, 5*time.Second, 10*time.Millisecond)
+	for _, call := range calls() {
+		want := testAccountID
+		if call.volume == "vol-none" {
+			want = ""
+		}
+		assert.Equal(t, want, call.account, "%s for %s", call.subject, call.volume)
+	}
+}
+
+// The provider's request span carries the account under the caller's trace,
+// with no mounter span of its own in between.
+func TestVolumeMounterAdapter_RequestSpanCarriesTheAccountUnderTheCaller(t *testing.T) {
+	sr := withRecordedSpans(t)
+	nc := connectMounterTest(t)
+	node := newMounterTestNode()
+	serveFakeNodeProvider(t, nc, node)
+	adapter := newNATSMounter(nc, node, nil)
+
+	ctx, caller := otel.Tracer("test").Start(t.Context(), "caller")
+	require.NoError(t, adapter.MountOne(ctx, testAccountID, &viperblocklegacyv1.EBSRequest{Name: "vol-acct"}))
+	caller.End()
+
+	request := clientSpan(t, sr)
+	assert.Equal(t, "ebs.volume.publish", request.Name())
+	assert.Equal(t, testAccountID, spanAccountOf(request), "the request span must name the account it acted for")
+	assert.Equal(t, caller.SpanContext().SpanID(), request.Parent().SpanID(), "the request span must join the caller directly")
+}
+
+// A detach spends part of its deadline on the unplug, so the seal must not
+// inherit what is left: a caller context already cancelled, or one shorter
+// than the seal, still gets the seal completed.
+func TestVolumeMounterAdapter_UnpublishOutlivesCallerContext(t *testing.T) {
+	const sealTime = 200 * time.Millisecond
+
+	nc := connectMounterTest(t)
+	node := newMounterTestNode()
+	fake := serveFakeNodeProvider(t, nc, node)
+	var sealed atomic.Int32
+	fake.setUnpublish(func(string) ebsprovider.UnpublishVolumeResponse {
+		time.Sleep(sealTime)
+		sealed.Add(1)
+		return ebsprovider.UnpublishVolumeResponse{}
+	})
+	volState := &recordingVolState{}
+	adapter := newNATSMounter(nc, node, volState)
+
+	short, cancelShort := context.WithTimeout(t.Context(), sealTime/4)
+	defer cancelShort()
+	require.NoError(t, adapter.UnmountOne(short, testAccountID, viperblocklegacyv1.EBSRequest{Name: "vol-detach"}))
+
+	cancelled, cancel := context.WithCancel(t.Context())
+	cancel()
+	require.NoError(t, adapter.Unmount(cancelled, mounterTestVM("vol-stop")))
+	assert.Equal(t, int32(2), sealed.Load())
+	assert.Equal(t, []string{"vol-stop:available"}, volState.snapshot())
+}
+
+// Abandon drops each export unsealed, at the version that carries NoSeal, and
+// tries every volume: an export torn down is logged, one not held here or
+// unknown to the provider is the ordinary case, and a refusal is returned.
+func TestVolumeMounterAdapter_AbandonUnpublishesWithoutSealing(t *testing.T) {
+	nc := connectMounterTest(t)
+	node := newMounterTestNode()
+	fake := serveFakeNodeProvider(t, nc, node)
+	fake.setUnpublish(func(volumeID string) ebsprovider.UnpublishVolumeResponse {
+		switch volumeID {
+		case "vol-torn":
+			return ebsprovider.UnpublishVolumeResponse{TornDown: true}
+		case "vol-unknown":
+			return unpublishRefusal(ebsprovider.ErrorCodeNotFound, "volume not found")
+		case "vol-refused":
+			return unpublishRefusal(ebsprovider.ErrorCodeUnavailable, "export busy")
+		}
+		return ebsprovider.UnpublishVolumeResponse{}
+	})
+	logs := captureSlogForTest(t)
+	adapter := newNATSMounter(nc, node, nil)
+
+	err := adapter.Abandon(t.Context(), mounterTestVM("vol-torn", "vol-absent", "vol-unknown", "vol-refused"), "superseded")
+	require.ErrorIs(t, err, ebsprovider.ErrUnavailable)
+	assert.Contains(t, err.Error(), "abandon vol-refused")
+	for _, volume := range []string{"vol-torn", "vol-absent", "vol-unknown"} {
+		assert.NotContains(t, err.Error(), volume)
+	}
+
+	calls := fake.snapshot()
+	require.Len(t, calls, 4, "every volume is attempted")
+	for _, call := range calls {
+		assert.True(t, call.noSeal, "%s must be dropped unsealed", call.volume)
+		assert.Equal(t, ebsprovider.NoSealSchemaVersion, call.version, "%s must be sent at the version carrying NoSeal", call.volume)
+	}
+	assert.Contains(t, err.Error(), "abandon vol-refused (superseded)", "the failure must say why the volume was abandoned")
+	const gaveUp = "Gave up a volume export"
+	assert.Contains(t, logs.String(), "instance=i-mounter volume=vol-torn reason=superseded")
+	assert.Equal(t, 1, strings.Count(logs.String(), gaveUp), "only a torn-down export is reported as given up")
+}
+
+// An abandon that finds no provider has not let go, so the caller is told.
+func TestVolumeMounterAdapter_AbandonNoResponderFails(t *testing.T) {
+	nc := connectMounterTest(t)
+	adapter := newNATSMounter(nc, newMounterTestNode(), nil)
+
+	err := adapter.Abandon(t.Context(), mounterTestVM("vol-1", "vol-2"), "superseded")
+	require.ErrorIs(t, err, nats.ErrNoResponders)
+	assert.Contains(t, err.Error(), "vol-1")
+	assert.Contains(t, err.Error(), "vol-2", "every volume is attempted")
+}
+
+// An abandon that is never answered gives up within its own bound rather
+// than the provider's data-movement budget for an unpublish.
+func TestVolumeMounterAdapter_AbandonGivesUpWithinItsBound(t *testing.T) {
+	const bound = 200 * time.Millisecond
+	nc := connectMounterTest(t)
+	node := newMounterTestNode()
+	spyOn(t, nc, "ebs.provider.v1."+node+".*")
+	adapter := newNATSMounter(nc, node, nil)
+	adapter.abandonTimeout = bound
+
+	done := make(chan error, 1)
+	go func() { done <- adapter.Abandon(t.Context(), mounterTestVM("vol-silent"), "superseded") }()
+	select {
+	case err := <-done:
+		require.ErrorIs(t, err, context.DeadlineExceeded)
+	case <-time.After(20 * bound):
+		t.Fatal("abandon outlasted its bound")
+	}
+}
+
+// The abandon span names the instance, the account and why it was abandoned,
+// and the provider's request spans sit under it.
+func TestVolumeMounterAdapter_AbandonSpanRecordsTheReason(t *testing.T) {
+	sr := withRecordedSpans(t)
+	nc := connectMounterTest(t)
+	node := newMounterTestNode()
+	serveFakeNodeProvider(t, nc, node)
+	adapter := newNATSMounter(nc, node, nil)
+
+	require.NoError(t, adapter.Abandon(t.Context(), mounterTestVM("vol-moved"), "superseded"))
+
+	request := clientSpan(t, sr)
+	var abandon sdktrace.ReadOnlySpan
+	for _, s := range sr.Ended() {
+		if s.Name() == "ebs.mounter.abandon" {
+			abandon = s
+		}
+	}
+	require.NotNil(t, abandon)
+	assert.Equal(t, "superseded", spanAttribute(abandon, "abandon.reason"))
+	assert.Equal(t, "i-mounter", spanAttribute(abandon, "instance.id"))
+	assert.Equal(t, testAccountID, spanAccountOf(abandon))
+	assert.Equal(t, abandon.SpanContext().SpanID(), request.Parent().SpanID())
+}

spinifex/runtime/daemon/vm_adapters_coverage_test.go

@@ -61,16 +61,6 @@ func closedAdapterTestConn(t *testing.T) *nats.Conn {
 	return nc
 }
 
-// respondJSON subscribes subject on nc and answers every request with reply.
-func respondJSON(t *testing.T, nc *nats.Conn, subject string, reply any) {
-	t.Helper()
-	data, err := json.Marshal(reply)
-	require.NoError(t, err)
-	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { _ = msg.Respond(data) })
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-}
-
 // --- stateStoreAdapter ---
 
 func TestStateStoreAdapter_DelegatesToJetStream(t *testing.T) {
@@ -190,35 +180,20 @@ func TestBuildVMManagerDeps_BackingStoreReady(t *testing.T) {
 		"no predastore configured and JetStream answering is ready")
 }
 
-// Variable stores live in the instance domain's bucket, read through the
-// same object store, with the live copy under the persistent data directory.
-func TestBuildVMManagerDeps_UEFIVarStore(t *testing.T) {
-	objects := objectstore.NewMemoryObjectStore()
-	d := &Daemon{
-		config:          &config.Config{DataDir: "/srv/spinifex", Predastore: config.PredastoreConfig{Bucket: "spinifex-instances"}},
-		vmMgr:           vm.NewManager(),
-		instanceObjects: objects,
-	}
-	got := d.buildVMManagerDeps().UEFIVarStore
-	assert.Same(t, objects, got.Objects)
-	assert.Equal(t, "spinifex-instances", got.Bucket)
-	assert.Equal(t, "/srv/spinifex", got.DataDir)
-
-	d.config.DataDir = ""
-	assert.Equal(t, DefaultDataDir, d.buildVMManagerDeps().UEFIVarStore.DataDir)
-}
-
 // --- volumeMounterAdapter ---
 
-func TestVolumeMounterAdapter_Mount_ClosedConnIsNotRetryable(t *testing.T) {
-	adapter := newVolumeMounterAdapter(closedAdapterTestConn(t), "node-closed", nil)
+// A connection that is down refuses the publish before sending it, so restore
+// backs off and retries, and there is no export to tear down.
+func TestVolumeMounterAdapter_Mount_DisconnectedIsRetryable(t *testing.T) {
+	adapter := newNATSMounter(closedAdapterTestConn(t), "node-closed", nil)
 	instance := &vm.VM{ID: "i-mount-closed"}
 	instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-closed"}}
 
 	err := adapter.Mount(t.Context(), instance)
 	require.ErrorIs(t, err, nats.ErrConnectionClosed)
-	assert.NotErrorIs(t, err, vm.ErrMountRetryable,
-		"only a missing responder means viperblockd is still starting")
+	require.ErrorIs(t, err, vm.ErrMountRetryable, "restore must back off until the connection is back")
+	assert.NotErrorIs(t, err, vm.ErrMountAmbiguous, "a request never sent left nothing to tear down")
+	require.ErrorIs(t, adapter.MountOne(t.Context(), testAccountID, &viperblocklegacyv1.EBSRequest{Name: "vol-hot"}), vm.ErrMountRetryable)
 }
 
 // recordingVolumeState records UpdateVolumeState calls and returns err.
@@ -237,7 +212,7 @@ func TestVolumeMounterAdapter_Unmount_Errors(t *testing.T) {
 
 	t.Run("no responder fails the seal and leaves the volume", func(t *testing.T) {
 		volState := &recordingVolumeState{}
-		adapter := newVolumeMounterAdapter(nc, uniqueAdapterID("node-unmount"), volState)
+		adapter := newNATSMounter(nc, uniqueAdapterID("node-unmount"), volState)
 		instance := &vm.VM{ID: "i-unmount-none"}
 		instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-a"}, {Name: "vol-b"}}
 
@@ -250,9 +225,9 @@ func TestVolumeMounterAdapter_Unmount_Errors(t *testing.T) {
 
 	t.Run("state update failure does not fail the unmount", func(t *testing.T) {
 		node := uniqueAdapterID("node-unmount")
-		respondJSON(t, nc, "ebs."+node+".unmount", viperblocklegacyv1.EBSUnMountResponse{Volume: "vol-a"})
+		serveFakeNodeProvider(t, nc, node)
 		volState := &recordingVolumeState{err: errInjected}
-		adapter := newVolumeMounterAdapter(nc, node, volState)
+		adapter := newNATSMounter(nc, node, volState)
 		instance := &vm.VM{ID: "i-unmount-state"}
 		instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-a"}}
 
@@ -261,74 +236,6 @@ func TestVolumeMounterAdapter_Unmount_Errors(t *testing.T) {
 	})
 }
 
-func TestVolumeMounterAdapter_Abandon(t *testing.T) {
-	nc := connectAdapterTestNATS(t, sharedNATSURL)
-
-	// abandonResponder answers per volume so one request can mix outcomes.
-	abandonResponder := func(t *testing.T, node string, replies map[string][]byte) *[]viperblocklegacyv1.VolumeAbandonRequest {
-		t.Helper()
-		var (
-			mu   sync.Mutex
-			seen []viperblocklegacyv1.VolumeAbandonRequest
-		)
-		sub, err := nc.Subscribe(viperblocklegacyv1.VolumeAbandonSubject(node), func(msg *nats.Msg) {
-			var req viperblocklegacyv1.VolumeAbandonRequest
-			_ = json.Unmarshal(msg.Data, &req)
-			mu.Lock()
-			seen = append(seen, req)
-			mu.Unlock()
-			_ = msg.Respond(replies[req.Volume])
-		})
-		require.NoError(t, err)
-		t.Cleanup(func() { _ = sub.Unsubscribe() })
-		return &seen
-	}
-	instanceWith := func(volumes ...string) *vm.VM {
-		v := &vm.VM{ID: "i-abandon", AccountID: testAccountID}
-		for _, name := range volumes {
-			v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{Name: name})
-		}
-		return v
-	}
-
-	t.Run("no responder", func(t *testing.T) {
-		adapter := newVolumeMounterAdapter(nc, uniqueAdapterID("node-abandon"), nil)
-		err := adapter.Abandon(t.Context(), instanceWith("vol-1"), "superseded")
-		assert.ErrorIs(t, err, nats.ErrNoResponders)
-	})
-
-	t.Run("outcomes per volume", func(t *testing.T) {
-		node := uniqueAdapterID("node-abandon")
-		seen := abandonResponder(t, node, map[string][]byte{
-			"vol-abandoned": []byte(`{"abandoned":true}`),
-			"vol-absent":    []byte(`{"abandoned":false}`),
-		})
-		adapter := newVolumeMounterAdapter(nc, node, nil)
-
-		require.NoError(t, adapter.Abandon(t.Context(), instanceWith("vol-abandoned", "vol-absent"), "superseded"),
-			"a volume not exported here is the ordinary case, not a failure")
-		require.Len(t, *seen, 2)
-		assert.Equal(t, viperblocklegacyv1.VolumeAbandonRequest{Volume: "vol-abandoned", Reason: "superseded"}, (*seen)[0])
-	})
-
-	t.Run("failures are joined and do not stop the sweep", func(t *testing.T) {
-		node := uniqueAdapterID("node-abandon")
-		seen := abandonResponder(t, node, map[string][]byte{
-			"vol-garbled": []byte(`not json`),
-			"vol-refused": []byte(`{"error":"export busy"}`),
-			"vol-ok":      []byte(`{"abandoned":true}`),
-		})
-		adapter := newVolumeMounterAdapter(nc, node, nil)
-
-		err := adapter.Abandon(t.Context(), instanceWith("vol-garbled", "vol-refused", "vol-ok"), "fenced")
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), "unmarshal abandon response for vol-garbled")
-		assert.Contains(t, err.Error(), "abandon vol-refused: export busy")
-		assert.NotContains(t, err.Error(), "vol-ok")
-		assert.Len(t, *seen, 3, "every volume is attempted")
-	})
-}
-
 // --- resourceControllerAdapter ---
 
 func TestResourceControllerAdapter_UnknownAndKnownType(t *testing.T) {

spinifex/runtime/daemon/vm_adapters_invariants_test.go

@@ -1,12 +1,10 @@
 package daemon
 
 import (
-	"encoding/json"
 	"testing"
 
 	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -23,18 +21,10 @@ import (
 func TestUnmountNeverReleasesBootVolume(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
 	volState := &recordingVolState{}
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, volState)
+	adapter := newNATSMounter(daemon.natsConn, daemon.node, volState)
 
-	// Stand in for the ebs daemon: every ebs.<node>.unmount succeeds (sealed).
-	sub, err := daemon.natsConn.Subscribe(viperblocklegacyv1.UnmountSubject(daemon.node), func(msg *nats.Msg) {
-		var req viperblocklegacyv1.EBSRequest
-		require.NoError(t, json.Unmarshal(msg.Data, &req))
-		data, err := json.Marshal(viperblocklegacyv1.EBSUnMountResponse{Volume: req.Name, Mounted: false})
-		require.NoError(t, err)
-		require.NoError(t, msg.Respond(data))
-	})
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
+	// Stand in for the provider: every unpublish succeeds (sealed).
+	serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
 
 	inst := &vm.VM{
 		ID: "i-unmount-boot",

spinifex/runtime/daemon/vm_adapters_test.go

@@ -2,7 +2,6 @@ package daemon
 
 import (
 	"context"
-	"encoding/json"
 	"reflect"
 	"testing"
 
@@ -234,118 +233,6 @@ func TestOnInstanceDownHook_OnlyRemovesTargetedInstance(t *testing.T) {
 	assert.False(t, dropPresent)
 }
 
-func TestVolumeMounterAdapter_MountOne(t *testing.T) {
-	tests := []struct {
-		name       string
-		responder  func(t *testing.T, msg *nats.Msg)
-		skipSub    bool
-		wantErr    bool
-		wantErrSub string
-		wantErrIs  error
-		wantNBDURI string
-		initialURI string
-	}{
-		{
-			name: "HappyPath_UpdatesNBDURI",
-			responder: func(t *testing.T, msg *nats.Msg) {
-				resp := viperblocklegacyv1.EBSMountResponse{URI: "nbd://mounted-vol"}
-				data, err := json.Marshal(resp)
-				require.NoError(t, err)
-				require.NoError(t, msg.Respond(data))
-			},
-			wantNBDURI: "nbd://mounted-vol",
-		},
-		{
-			name:       "NATSNoResponders_ReturnsError",
-			skipSub:    true,
-			wantErr:    true,
-			wantErrSub: "ebs.mount NATS request",
-		},
-		{
-			name: "UnmarshalFailure_ReturnsError",
-			responder: func(t *testing.T, msg *nats.Msg) {
-				require.NoError(t, msg.Respond([]byte("not json")))
-			},
-			wantErr:    true,
-			wantErrSub: "unmarshal ebs.mount response",
-		},
-		{
-			name: "ResponseError_IncludedInError",
-			responder: func(t *testing.T, msg *nats.Msg) {
-				resp := viperblocklegacyv1.EBSMountResponse{Error: "boom"}
-				data, err := json.Marshal(resp)
-				require.NoError(t, err)
-				require.NoError(t, msg.Respond(data))
-			},
-			wantErr:    true,
-			wantErrSub: "boom",
-		},
-		{
-			name: "EmptyURI_ReturnsErrMountAmbiguous",
-			responder: func(t *testing.T, msg *nats.Msg) {
-				resp := viperblocklegacyv1.EBSMountResponse{URI: ""}
-				data, err := json.Marshal(resp)
-				require.NoError(t, err)
-				require.NoError(t, msg.Respond(data))
-			},
-			wantErr:   true,
-			wantErrIs: vm.ErrMountAmbiguous,
-		},
-		{
-			name: "EmptyURI_PreservesInitialNBDURIOnFailure",
-			responder: func(t *testing.T, msg *nats.Msg) {
-				resp := viperblocklegacyv1.EBSMountResponse{URI: ""}
-				data, err := json.Marshal(resp)
-				require.NoError(t, err)
-				require.NoError(t, msg.Respond(data))
-			},
-			initialURI: "nbd://stale",
-			wantErr:    true,
-			wantErrIs:  vm.ErrMountAmbiguous,
-			wantNBDURI: "nbd://stale",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			daemon := createTestDaemon(t, sharedNATSURL)
-			adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
-
-			if !tt.skipSub {
-				sub, err := daemon.natsConn.Subscribe("ebs.node-1.mount", func(msg *nats.Msg) {
-					tt.responder(t, msg)
-				})
-				require.NoError(t, err)
-				defer sub.Unsubscribe()
-			}
-
-			req := &viperblocklegacyv1.EBSRequest{
-				Name:       "vol-mountone",
-				DeviceName: "/dev/sdf",
-				NBDURI:     tt.initialURI,
-			}
-
-			err := adapter.MountOne(t.Context(), "", req)
-			if tt.wantErr {
-				require.Error(t, err)
-				if tt.wantErrSub != "" {
-					assert.Contains(t, err.Error(), tt.wantErrSub)
-				}
-				if tt.wantErrIs != nil {
-					assert.ErrorIs(t, err, tt.wantErrIs)
-				}
-				assert.Equal(t, tt.wantNBDURI, req.NBDURI,
-					"NBDURI must not be overwritten on failure")
-				return
-			}
-
-			require.NoError(t, err)
-			assert.Equal(t, tt.wantNBDURI, req.NBDURI,
-				"happy path must write resolved NBDURI back to req")
-		})
-	}
-}
-
 // --- ReleaseGPU ---
 
 // ReleaseGPU is a no-op when the daemon has no GPU manager.
@@ -674,56 +561,9 @@ func TestInstanceCleanerAdapter_ReleaseAttachedENIs_ListInstanceENIsErrorTolerat
 		"an enumeration failure must not surface as a terminate error")
 }
 
-// TestVolumeMounterAdapter_Mount_WrapsErrMountRetryable verifies that a
-// viperblockd ebs.mount response with Retryable=true is wrapped so
-// vm.Manager's recovery-relaunch retry can match it with errors.Is. A
-// Retryable=false response must NOT match, so relaunchAll still fails fast
-// on permanent mount errors.
-func TestVolumeMounterAdapter_Mount_WrapsErrMountRetryable(t *testing.T) {
-	daemon := createTestDaemon(t, sharedNATSURL)
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
-
-	sub, err := daemon.natsConn.Subscribe("ebs."+daemon.node+".mount", func(msg *nats.Msg) {
-		resp := viperblocklegacyv1.EBSMountResponse{Error: "state not found", Retryable: true}
-		data, _ := json.Marshal(resp)
-		_ = msg.Respond(data)
-	})
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	instance := &vm.VM{ID: "i-mount-retryable"}
-	instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-retryable"}}
-
-	err = adapter.Mount(t.Context(), instance)
-	require.Error(t, err)
-	assert.ErrorIs(t, err, vm.ErrMountRetryable,
-		"a Retryable mount response must wrap vm.ErrMountRetryable so relaunchAll can retry it")
-}
-
-func TestVolumeMounterAdapter_Mount_PermanentErrorNotWrapped(t *testing.T) {
-	daemon := createTestDaemon(t, sharedNATSURL)
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
-
-	sub, err := daemon.natsConn.Subscribe("ebs."+daemon.node+".mount", func(msg *nats.Msg) {
-		resp := viperblocklegacyv1.EBSMountResponse{Error: "volume vol-permanent is already mounted read_only=true on this node"}
-		data, _ := json.Marshal(resp)
-		_ = msg.Respond(data)
-	})
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	instance := &vm.VM{ID: "i-mount-permanent"}
-	instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-permanent"}}
-
-	err = adapter.Mount(t.Context(), instance)
-	require.Error(t, err)
-	assert.NotErrorIs(t, err, vm.ErrMountRetryable,
-		"a non-Retryable mount response must not be mistaken for a transient failure")
-}
-
 func TestVolumeMounterAdapter_Mount_NoResponderIsRetryable(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
+	adapter := newNATSMounter(daemon.natsConn, daemon.node, nil)
 
 	// No subscriber on the mount subject: viperblockd is still starting, so
 	// the request returns nats.ErrNoResponders, which must be retryable.

spinifex/runtime/daemon/vm_adapters_trace_test.go

@@ -1,12 +1,10 @@
 package daemon
 
 import (
-	"encoding/json"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"testing"
+	"time"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -37,70 +35,65 @@ func withRecordedSpans(t *testing.T) *tracetest.SpanRecorder {
 	return sr
 }
 
-// TestEbsRequestWithTrace_HeaderCarriesTraceparent pins the minimum bar: the
-// outbound ebs.mount NATS message carries a non-empty traceparent header, so
-// a consumer that calls natsmsg.ExtractTraceContext has something to join.
-func TestEbsRequestWithTrace_HeaderCarriesTraceparent(t *testing.T) {
-	withRecordedSpans(t)
-	daemon := createTestDaemon(t, sharedNATSURL)
-
+// requestOnceOn answers a single request on subject with an empty JSON body
+// and hands back the headers it carried, so a test can assert on the wire.
+func requestOnceOn(t *testing.T, nc *nats.Conn, subject string) <-chan nats.Header {
+	t.Helper()
 	headerCh := make(chan nats.Header, 1)
-	sub, err := daemon.natsConn.Subscribe("ebs.node-1.mount", func(msg *nats.Msg) {
+	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
 		headerCh <- msg.Header
-		resp := viperblocklegacyv1.EBSMountResponse{URI: "nbd://traced-vol"}
-		data, marshalErr := json.Marshal(resp)
-		require.NoError(t, marshalErr)
-		require.NoError(t, msg.Respond(data))
+		require.NoError(t, msg.Respond([]byte("{}")))
 	})
 	require.NoError(t, err)
-	defer sub.Unsubscribe()
+	require.NoError(t, nc.Flush())
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	return headerCh
+}
 
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
-	req := &viperblocklegacyv1.EBSRequest{Name: "vol-traced", DeviceName: "/dev/sdf"}
-	require.NoError(t, adapter.MountOne(t.Context(), "", req))
+// TestEbsRequestWithTrace_HeaderCarriesTraceparent pins the minimum bar: the
+// outbound ebs.* NATS message carries a non-empty traceparent header, so a
+// consumer that calls natsmsg.ExtractTraceContext has something to join.
+func TestEbsRequestWithTrace_HeaderCarriesTraceparent(t *testing.T) {
+	withRecordedSpans(t)
+	nc := connectMounterTest(t)
+	subject := "ebs." + newMounterTestNode() + ".delete"
+	headerCh := requestOnceOn(t, nc, subject)
 
-	hdr := <-headerCh
-	assert.NotEmpty(t, hdr.Get("traceparent"), "outbound ebs.mount request must carry a traceparent header")
+	_, err := ebsRequestWithTrace(t.Context(), nc, "", subject, []byte("{}"), 5*time.Second)
+	require.NoError(t, err)
+
+	assert.NotEmpty(t, (<-headerCh).Get("traceparent"), "outbound ebs.* request must carry a traceparent header")
 }
 
 // TestEbsRequestWithTrace_ProducerConsumerLinked proves the actual parent/child
-// linkage: the consumer span opened via natsmsg.StartConsumerSpan (the same
-// helper viperblockd's ebs.mount/ebs.unmount handlers use) shares the producer
-// span's trace ID and names it as its parent, rather than rooting a new trace
-// — the exact gap this fix closes for the 81 mount + 81 unmount rooted spans
-// the trace analysis found.
+// linkage: the consumer span opened via natsmsg.StartConsumerSpan (the helper
+// viperblockd's legacy handlers use) shares the producer span's trace ID and
+// names it as its parent, rather than rooting a new trace.
 func TestEbsRequestWithTrace_ProducerConsumerLinked(t *testing.T) {
 	sr := withRecordedSpans(t)
-	daemon := createTestDaemon(t, sharedNATSURL)
+	nc := connectMounterTest(t)
+	subject := "ebs." + newMounterTestNode() + ".delete"
 
 	consumerEnded := make(chan struct{})
-	sub, err := daemon.natsConn.Subscribe("ebs.node-1.unmount", func(msg *nats.Msg) {
+	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
 		defer close(consumerEnded)
-		// Mirrors viperblockd's ebs.unmount handler: open a consumer span
-		// joining whatever trace context the producer injected.
 		_, span := natsmsg.StartConsumerSpan(msg)
 		defer span.End()
-
-		resp := viperblocklegacyv1.EBSUnMountResponse{}
-		data, marshalErr := json.Marshal(resp)
-		require.NoError(t, marshalErr)
-		require.NoError(t, msg.Respond(data))
+		require.NoError(t, msg.Respond([]byte("{}")))
 	})
 	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
 	defer sub.Unsubscribe()
 
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
-	require.NoError(t, adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-linked"}))
+	_, err = ebsRequestWithTrace(t.Context(), nc, "", subject, []byte("{}"), 5*time.Second)
+	require.NoError(t, err)
 	// The handler replies before its deferred span.End runs, so the consumer
-	// span may not be recorded yet when UnmountOne returns.
+	// span may not be recorded yet when the request returns.
 	<-consumerEnded
 
 	spans := sr.Ended()
 	require.Len(t, spans, 2, "expected one producer span and one consumer span")
 
-	// Both spans share the "NATS ebs.node-1.unmount" name; distinguish by kind
-	// (producer = client, consumer = server, as set by ebsRequestWithTrace and
-	// natsmsg.StartConsumerSpan respectively).
 	var producer, consumer sdktrace.ReadOnlySpan
 	for _, s := range spans {
 		switch s.SpanKind() {
@@ -119,43 +112,21 @@ func TestEbsRequestWithTrace_ProducerConsumerLinked(t *testing.T) {
 		"consumer span's parent must be the producer span")
 }
 
-// mountOnceOn answers a single ebs.mount on subject and hands back the headers
-// the adapter sent, so a test can assert on the wire rather than the span only.
-func mountOnceOn(t *testing.T, nc *nats.Conn, subject string) <-chan nats.Header {
-	t.Helper()
-	headerCh := make(chan nats.Header, 1)
-	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
-		headerCh <- msg.Header
-		data, marshalErr := json.Marshal(viperblocklegacyv1.EBSMountResponse{URI: "nbd://vol"})
-		require.NoError(t, marshalErr)
-		require.NoError(t, msg.Respond(data))
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-	return headerCh
-}
-
 // The ebs hop used to root its own trace from context.Background(), so block
 // storage work appeared as an orphan trace no request could be followed into.
 // Given a caller's context it must extend that trace instead.
 func TestEbsRequestWithTrace_JoinsTheCallersTrace(t *testing.T) {
 	sr := withRecordedSpans(t)
-	daemon := createTestDaemon(t, sharedNATSURL)
-	mountOnceOn(t, daemon.natsConn, "ebs.node-1.mount")
+	nc := connectMounterTest(t)
+	subject := "ebs." + newMounterTestNode() + ".delete"
+	requestOnceOn(t, nc, subject)
 
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
 	ctx, caller := otel.Tracer("test").Start(t.Context(), "caller")
-	require.NoError(t, adapter.MountOne(ctx, "000000000042", &viperblocklegacyv1.EBSRequest{Name: "vol-ctx"}))
+	_, err := ebsRequestWithTrace(ctx, nc, "000000000042", subject, []byte("{}"), 5*time.Second)
+	require.NoError(t, err)
 	caller.End()
 
-	var producer sdktrace.ReadOnlySpan
-	for _, s := range sr.Ended() {
-		if s.SpanKind() == trace.SpanKindClient {
-			producer = s
-		}
-	}
-	require.NotNil(t, producer, "producer span must be recorded")
-
+	producer := clientSpan(t, sr)
 	assert.Equal(t, caller.SpanContext().TraceID(), producer.SpanContext().TraceID(),
 		"the ebs span must join the caller's trace rather than root one of its own")
 	assert.Equal(t, caller.SpanContext().SpanID(), producer.Parent().SpanID(),
@@ -167,25 +138,16 @@ func TestEbsRequestWithTrace_JoinsTheCallersTrace(t *testing.T) {
 // reads the header to attribute its own consumer span.
 func TestEbsRequestWithTrace_CarriesTheAccount(t *testing.T) {
 	sr := withRecordedSpans(t)
-	daemon := createTestDaemon(t, sharedNATSURL)
-	headerCh := mountOnceOn(t, daemon.natsConn, "ebs.node-1.mount")
+	nc := connectMounterTest(t)
+	subject := "ebs." + newMounterTestNode() + ".delete"
+	headerCh := requestOnceOn(t, nc, subject)
 
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
-	instance := &vm.VM{ID: "i-acct", AccountID: "000000000042"}
-	instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-acct"}}
-	require.NoError(t, adapter.Mount(t.Context(), instance))
+	_, err := ebsRequestWithTrace(t.Context(), nc, "000000000042", subject, []byte("{}"), 5*time.Second)
+	require.NoError(t, err)
 
 	assert.Equal(t, "000000000042", (<-headerCh).Get(natsmsg.AccountIDHeader),
 		"viperblockd reads the account off the header, so it must be set")
-
-	var producer sdktrace.ReadOnlySpan
-	for _, s := range sr.Ended() {
-		if s.SpanKind() == trace.SpanKindClient {
-			producer = s
-		}
-	}
-	require.NotNil(t, producer)
-	assert.Equal(t, "000000000042", spanAccountOf(producer),
+	assert.Equal(t, "000000000042", spanAccountOf(clientSpan(t, sr)),
 		"the ebs span must name the account it acted for")
 }
 
@@ -193,25 +155,39 @@ func TestEbsRequestWithTrace_CarriesTheAccount(t *testing.T) {
 // would be worse than leaving it unattributed.
 func TestEbsRequestWithTrace_OmitsAnAbsentAccount(t *testing.T) {
 	sr := withRecordedSpans(t)
-	daemon := createTestDaemon(t, sharedNATSURL)
-	headerCh := mountOnceOn(t, daemon.natsConn, "ebs.node-1.mount")
+	nc := connectMounterTest(t)
+	subject := "ebs." + newMounterTestNode() + ".delete"
+	headerCh := requestOnceOn(t, nc, subject)
 
-	adapter := newVolumeMounterAdapter(daemon.natsConn, daemon.node, nil)
-	require.NoError(t, adapter.MountOne(t.Context(), "", &viperblocklegacyv1.EBSRequest{Name: "vol-none"}))
+	_, err := ebsRequestWithTrace(t.Context(), nc, "", subject, []byte("{}"), 5*time.Second)
+	require.NoError(t, err)
 
 	assert.Empty(t, (<-headerCh).Get(natsmsg.AccountIDHeader),
 		"an unattributed request must not carry a blank account header")
+	assert.Empty(t, spanAccountOf(clientSpan(t, sr)), "an unattributed span must carry no account")
+}
+
+// clientSpan returns the one client span sr recorded.
+func clientSpan(t *testing.T, sr *tracetest.SpanRecorder) sdktrace.ReadOnlySpan {
+	t.Helper()
+	var spans []sdktrace.ReadOnlySpan
 	for _, s := range sr.Ended() {
 		if s.SpanKind() == trace.SpanKindClient {
-			assert.Empty(t, spanAccountOf(s), "an unattributed span must carry no account")
+			spans = append(spans, s)
 		}
 	}
+	require.Len(t, spans, 1)
+	return spans[0]
 }
 
 // spanAccountOf returns the account attribute of span, or "".
 func spanAccountOf(span sdktrace.ReadOnlySpan) string {
+	return spanAttribute(span, natsmsg.AttrAccountID)
+}
+
+func spanAttribute(span sdktrace.ReadOnlySpan, key string) string {
 	for _, attr := range span.Attributes() {
-		if string(attr.Key) == natsmsg.AttrAccountID {
+		if string(attr.Key) == key {
 			return attr.Value.AsString()
 		}
 	}

spinifex/runtime/roles/qemunbdd/provider.go

@@ -533,6 +533,11 @@ func (p *Provider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSna
 		return nil, fmt.Errorf("%w: volume %s", ebsprovider.ErrNotFound, req.VolumeID)
 	}
 
+	// qemu-nbd holds a published volume's file locked and keeps writing it, so
+	// no copy of it is point-in-time. The volume must be unpublished first.
+	if pub, ok := p.published[req.VolumeID]; ok && !pub.dead() {
+		return nil, fmt.Errorf("%w: volume %s is published, so it cannot be snapshotted until it is unpublished", ebsprovider.ErrVolumeInUse, req.VolumeID)
+	}
 	if _, err := p.run.Run(ctx, "qemu-img", "convert", "-O", "qcow2", volPath, snapPath); err != nil {
 		return nil, fmt.Errorf("create snapshot %s: %w", req.SnapshotID, err)
 	}

spinifex/runtime/roles/qemunbdd/provider_test.go

@@ -414,6 +414,30 @@ func TestCreateSnapshot_Argv(t *testing.T) {
 	assert.Equal(t, []string{"info", "--output=json", "--force-share", snapPath}, calls[1].args)
 }
 
+// A published volume is refused before qemu-img runs, and an unpublished one
+// snapshots again from its file.
+func TestCreateSnapshot_PublishedVolumeIsRefused(t *testing.T) {
+	p, fr := newTestProvider(t)
+	ctx := context.Background()
+	volPath := p.volumePath("vol-snap-pub")
+	require.NoError(t, os.WriteFile(volPath, nil, 0o640))
+	_, err := p.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-snap-pub", NodeID: "node-1"})
+	require.NoError(t, err)
+	published := len(fr.callsSnapshot())
+
+	_, err = p.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-pub", VolumeID: "vol-snap-pub"})
+	require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse)
+	assert.Len(t, fr.callsSnapshot(), published, "a refused snapshot must not run qemu-img")
+	_, statErr := os.Stat(p.snapshotPath("snap-pub"))
+	require.ErrorIs(t, statErr, os.ErrNotExist)
+
+	_, err = p.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: versioned(), VolumeID: "vol-snap-pub", NodeID: "node-1"})
+	require.NoError(t, err)
+	snap, err := p.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-pub", VolumeID: "vol-snap-pub"})
+	require.NoError(t, err)
+	assert.Equal(t, "vol-snap-pub", snap.SourceVolumeID)
+}
+
 func TestCreateSnapshot_NotFoundSourceVolume(t *testing.T) {
 	p, _ := newTestProvider(t)
 	_, err := p.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{Versioned: versioned(), SnapshotID: "snap-orphan", VolumeID: "vol-never-existed"})

spinifex/runtime/roles/qemunbdd/snapshot_published_test.go

@@ -0,0 +1,104 @@
+package qemunbdd_test
+
+import (
+	"bufio"
+	"io"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// attachedClient stands in for a guest's QEMU: a qemu-io holding a connection
+// to the export open, so the snapshot runs against a volume in use.
+type attachedClient struct {
+	cmd   *exec.Cmd
+	stdin io.WriteCloser
+	out   *bufio.Scanner
+}
+
+func attachClient(t *testing.T, uri string) *attachedClient {
+	t.Helper()
+	cmd := exec.Command("qemu-io", "-f", "raw", uri)
+	stdin, err := cmd.StdinPipe()
+	require.NoError(t, err)
+	stdout, err := cmd.StdoutPipe()
+	require.NoError(t, err)
+	cmd.Stderr = cmd.Stdout
+	require.NoError(t, cmd.Start())
+	c := &attachedClient{cmd: cmd, stdin: stdin, out: bufio.NewScanner(stdout)}
+	t.Cleanup(c.detach)
+	return c
+}
+
+// run sends one qemu-io command and returns the line reporting its result.
+func (c *attachedClient) run(t *testing.T, command, want string) {
+	t.Helper()
+	_, err := io.WriteString(c.stdin, command+"\n")
+	require.NoError(t, err)
+	done := make(chan string, 1)
+	go func() {
+		for c.out.Scan() {
+			if line := c.out.Text(); strings.Contains(line, want) || strings.Contains(line, "failed") {
+				done <- line
+				return
+			}
+		}
+		done <- "qemu-io exited"
+	}()
+	select {
+	case line := <-done:
+		require.Contains(t, line, want)
+	case <-time.After(10 * time.Second):
+		t.Fatalf("qemu-io did not answer %q", command)
+	}
+}
+
+func (c *attachedClient) detach() {
+	_, _ = io.WriteString(c.stdin, "quit\n")
+	_ = c.stdin.Close()
+	_ = c.cmd.Wait()
+}
+
+// A snapshot of a volume a guest is attached to is refused and writes nothing,
+// leaving the guest's connection intact. Once the volume is unpublished it
+// snapshots, holding what the guest wrote.
+func TestQEMUNBDSnapshotOfAttachedVolumeIsRefused(t *testing.T) {
+	requireQEMUTools(t)
+	provider := newQEMUProvider(t)
+	ctx := t.Context()
+	const volumeID, snapshotID = "vol-attached-snap", "snap-attached-snap"
+	_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 64 << 20}})
+	require.NoError(t, err)
+	pub, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: "node-1"})
+	require.NoError(t, err)
+
+	guest := attachClient(t, pub.Export.NBD.NBDURI)
+	guest.run(t, "write -P 0xab 1M 64k", "wrote 65536/65536")
+
+	_, err = provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID})
+	require.ErrorIs(t, err, ebsprovider.ErrVolumeInUse)
+	listed, err := provider.ListSnapshots(ctx, ebsprovider.ListSnapshotsRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	for _, snap := range listed.Snapshots {
+		assert.NotEqual(t, snapshotID, snap.ID, "a refused snapshot must leave nothing behind")
+	}
+	guest.run(t, "read -P 0xab 1M 64k", "read 65536/65536")
+
+	guest.detach()
+	_, err = provider.UnpublishVolume(ctx, ebsprovider.UnpublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: "node-1"})
+	require.NoError(t, err)
+	snap, err := provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID})
+	require.NoError(t, err)
+	require.Equal(t, ebsprovider.SnapshotStateCompleted, snap.State)
+
+	snapPath := strings.TrimPrefix(snap.Handle, "file://")
+	out, err := exec.Command("qemu-io", "-f", "qcow2", "-r", "-c", "read -P 0xab 1M 64k", snapPath).CombinedOutput()
+	require.NoError(t, err, "%s", out)
+	assert.Contains(t, string(out), "read 65536/65536")
+	assert.NotContains(t, string(out), "Pattern verification failed")
+}

tests/e2e/single/createimage_test.go

@@ -72,10 +72,9 @@ func runCreateImage(t *testing.T, fix *Fixture) {
 
 	runBootVerifyAMI(t, fix, customAMIID, "running-path")
 
-	// The offline path: CreateImage against the same instance stopped. The
-	// daemon's IsRunning decision is the instance's actual VM status at call
-	// time (daemon_handlers_image.go), not the NoReboot flag, so stopping the
-	// source instance is what selects snapshotStoppedVolume.
+	// The offline path: CreateImage against the same instance stopped. Both
+	// states take the single snapshotVolume path, so this proves the provider
+	// snapshots a volume with no connected client as well as a live one.
 	harness.Step(t, "stop-instances %s (precondition for stopped-path CreateImage)", instanceID)
 	_, err = fix.AWS.EC2.StopInstances(&ec2.StopInstancesInput{
 		InstanceIds: []*string{aws.String(instanceID)},