test(lambda): sample heap only across the measured lifetimes and join the sampler

The sampler now starts after the post-GC baseline and stops after both
replies are read, and cleanup joins it if the test fails first.

Makefile

@@ -90,6 +90,7 @@ go_build:
 LAMBDA_TESTLINK_DIR := ./bin/lambda-testlink
 LAMBDA_TESTLINK_PKG := github.com/mulgadc/spinifex/spinifex/runtime/daemon
 LAMBDA_TESTLINK_PIN := $(LAMBDA_TESTLINK_PKG).lambdaHardKillSHA256
+LAMBDA_TESTLINK_GW := github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw
 LAMBDA_DIGEST_RE := ^[0-9a-f]{64}$$
 build-lambda-testlink: override export LAMBDA_IMAGE_DIGEST := $(value LAMBDA_IMAGE_DIGEST)
 build-lambda-testlink:
@@ -104,8 +105,8 @@ build-lambda-testlink:
 	@digest=$$(sha256sum $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so | cut -d' ' -f1); \
 	test $${#digest} -eq 64 || { echo "initializer digest missing"; exit 1; }; \
 	echo "initializer sha256 $$digest"; \
-	pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
-	LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ && \
+	pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST -X $(LAMBDA_TESTLINK_GW).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
+	LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ ./spinifex/runtime/roles/awsgw/ && \
 	go build $(GO_BUILD_MOD) -tags lambdatestlink -ldflags "$(subst -s -w ,,$(LDFLAGS)) $$pin" -o $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 	@mkdir -p $(LAMBDA_TESTLINK_DIR)
 	mv $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) $(LAMBDA_TESTLINK_DIR)/

contracts/cluster/v1/heartbeat.go

@@ -1,9 +1,6 @@
 package clusterv1
 
-import (
-	"encoding/json"
-	"time"
-)
+import "time"
 
 // ClusterStateBucket is the KV bucket holding each daemon's heartbeat alongside
 // other cluster-wide state.
@@ -14,27 +11,6 @@ func HeartbeatKey(node string) string {
 	return "heartbeat." + node
 }
 
-// LivenessWindow is how recently a node must have heartbeated for a reader
-// deciding on a takeover to count it live. It is six heartbeats, compared
-// across node clocks, where NTP skew is small against it.
-const LivenessWindow = 60 * time.Second
-
-// LiveAt reports whether a heartbeat written at written still vouches for its
-// node at now: it does while it is under LivenessWindow old.
-func LiveAt(written, now time.Time) bool {
-	return now.Sub(written) < LivenessWindow
-}
-
-// HeartbeatWritten dates a stored heartbeat value. One that cannot be decoded
-// or dated vouches for nothing, which is an answer rather than a fault.
-func HeartbeatWritten(data []byte) (time.Time, bool) {
-	var beat HeartbeatTimestamp
-	if json.Unmarshal(data, &beat) != nil {
-		return time.Time{}, false
-	}
-	return beat.Time()
-}
-
 // HeartbeatTimestamp is the read side of a heartbeat for node liveness. It
 // decodes only the timestamp, so a reader does not depend on the capacity
 // figures the daemon writes beside it.

contracts/cluster/v1/heartbeat_test.go

@@ -33,23 +33,3 @@ func TestHeartbeatTimestamp_UndatedBeatVouchesForNothing(t *testing.T) {
 		assert.False(t, ok, "timestamp %q", ts)
 	}
 }
-
-// A beat is live strictly under the window, so one exactly the window old is
-// the first a takeover may treat as stopped.
-func TestLiveAt_IsStrictlyUnderTheWindow(t *testing.T) {
-	now := time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)
-	assert.True(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow+time.Nanosecond), now))
-	assert.False(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow), now))
-	assert.True(t, clusterv1.LiveAt(now.Add(time.Second), now), "a beat from a clock running ahead is live")
-}
-
-func TestHeartbeatWritten_DatesAStoredBeatAndRefusesAnUndecodableOne(t *testing.T) {
-	written, ok := clusterv1.HeartbeatWritten([]byte(`{"node":"node-a","timestamp":"2026-10-30T01:02:03Z"}`))
-	require.True(t, ok)
-	assert.True(t, written.Equal(time.Date(2026, 10, 30, 1, 2, 3, 0, time.UTC)))
-
-	for _, raw := range []string{"not json", `{"node":"node-a"}`} {
-		_, ok := clusterv1.HeartbeatWritten([]byte(raw))
-		assert.False(t, ok, raw)
-	}
-}

contracts/lambda/v1/README.md

@@ -22,6 +22,11 @@ Tenant code may be able to reach the listener, but it cannot open the agent's co
 For each generation the executor creates ephemeral self-signed host and guest certificates whose single URI SAN is `spinifex-lambda://<Region>/<account>/<function ID>/<environment ID>/<generation>/<role>`, with role `host` or `guest`.
 Each end validates the peer certificate against the one pinned peer certificate and separately requires that URI SAN with the opposite role; hostname verification is not relied on.
 ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/2` on both host and guest, whose first frame carries the execution-role credential, so a test-link end and a release end fail the handshake; release builds are unchanged.
+In the same builds, the gateway reaches the host daemon on the local abstract socket `spinifex-lambda-testlink-control/3`, and each request carries the runtime-image digest the gateway was built with; the daemon refuses a missing or different digest before starting the attempt, and a gateway and daemon on different socket names never connect.
+On that socket a request's raw payload is at most 6 MiB, checked by the gateway before encoding and by the daemon after decoding.
+Its encoded frame is at most the base64 length of 6 MiB plus 64 KiB of metadata headroom and 64 bytes of field slack, and a receiver checks the frame length before allocating; metadata has no separate bound within the frame.
+The credential travels as its own frame of at most 8 KiB, and the reply has the same payload and frame bounds as the request.
+The daemon holds at most two connections at once, from accept to close, and leaves the rest unaccepted in the listen backlog.
 Under TLS 1.3 the guest's handshake can complete before the host refuses its certificate, so the guest treats the connection as accepted only after the host's first message.
 
 **Bootstrap material:** the executor passes these fw_cfg entries under `opt/spinifex/lambda/`, and the agent reads them as root and unloads `qemu_fw_cfg` before user code starts.

docs/DESIGN.md

@@ -111,9 +111,9 @@ Daemons (`spinifex/runtime/daemon/daemon.go`) subscribe to NATS topics and handl
 When a daemon handles `RunInstances`:
 
 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
-2. **Volume Generation**: Creates the boot volume through the EBS provider and, for a UEFI guest, seeds its variable store as an object under `uefi/<instance-id>/` with a store record in `spinifex-uefi-stores` that points at it
-   At launch a UEFI instance's `vars.fd` is fetched from the object its store pointer names (or reused when `seeded.json` on this node already names the same object) after a firmware architecture and VARS size check, and a UEFI instance with no live store record naming a pointer, or one that still carries an EFI volume request, is refused.
-   The store is written back to the object store on write and at stop, and a terminated instance's store is removed by a cluster-wide reaper.
+2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
+   A UEFI instance whose record carries a variable store pointer has no EFI volume: its `vars.fd` is fetched from the object under `uefi/` in the configured Predastore bucket that the pointer names, recorded in the `spinifex-uefi-stores` JetStream bucket (or reused when `seeded.json` on this node already names the same object) after a firmware architecture and VARS size check, and a record with both or neither is refused.
+   RunInstances still creates the EFI volume, nothing writes `vars.fd` back to the object store yet, and a terminated instance's store is removed by a cluster-wide reaper.
 3. **Volume Mount**: Publishes each volume through the EBS provider contract (`PublishVolume` on `ebs.provider.v1.{node}.mount`), receives its export
 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management

spinifex/agents/lambda/hostlink/test_stream.go

@@ -321,6 +321,19 @@ func (s *TestStream) HostSend(req Request) error {
 	return s.write(frame{Type: typeInvoke, Request: &req})
 }
 
+// CheckInvokeFits encodes req as HostSend would and returns ErrFrameTooLarge
+// when the frame exceeds the bound, so a caller can refuse before launch.
+func CheckInvokeFits(req Request) error {
+	b, err := json.Marshal(frame{Type: typeInvoke, Request: &req})
+	if err != nil {
+		return err
+	}
+	if len(b) > maxFrame {
+		return ErrFrameTooLarge
+	}
+	return nil
+}
+
 // HostShutdown writes a shutdown frame, as the host side of the carrier.
 func (s *TestStream) HostShutdown() error {
 	return s.write(frame{Type: typeShutdown})

spinifex/domains/acm/store.go

@@ -14,6 +14,7 @@ import (
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -183,7 +184,7 @@ func NewStore(ctx context.Context, nc *nats.Conn, masterKey []byte) (*Store, err
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketACM, KVBucketACMVersion)
+	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketACM, KVBucketACMVersion)
 	if err != nil {
 		return nil, err
 	}

spinifex/domains/admission/quota/bedrock.go

@@ -10,7 +10,7 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 	"golang.org/x/time/rate"
 )
@@ -171,7 +171,7 @@ func (s *Service) RunBedrockRPMSync(ctx context.Context, js jetstream.JetStream)
 	if s == nil || !s.limits.RequestsPerMinuteEnabled {
 		return
 	}
-	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketBedrockRPMUsage, 1)
+	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketBedrockRPMUsage, 1)
 	if err != nil {
 		slog.Warn("bedrock rpm sync: open usage bucket failed, sync disabled", "err", err)
 		return

spinifex/domains/admission/quota/override.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -187,7 +187,7 @@ func (s *Service) PutAccountQuota(ctx context.Context, accountID string, over Ov
 	over.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
 	over.UpdatedBy = updatedBy
 
-	_, err := kvstore.Update(ctx, s.overrides, accountID, kvstore.CASConfig{
+	_, err := kvutil.Update(ctx, s.overrides, accountID, kvutil.CASConfig{
 		CreateIfAbsent: true,
 		Attempts:       overrideMaxAttempts,
 		Exhausted:      func(string, int) error { return errOverridesContended },
@@ -230,7 +230,7 @@ func (s *Service) clearAccountQuota(ctx context.Context, accountID string) error
 			return nil
 		case errors.Is(err, jetstream.ErrKeyNotFound):
 			return nil
-		case kvstore.IsRevisionConflict(err):
+		case kvutil.IsRevisionConflict(err):
 			// A concurrent write landed between the read and the guarded
 			// delete; retry against whatever revision now holds.
 			continue

spinifex/domains/admission/quota/quota.go

@@ -22,10 +22,10 @@ const ReconcileInterval = 30 * time.Second
 // dimension; the live-counted dimensions need no stored state.
 const KVBucketAccountUsage = "spinifex-account-usage"
 
-// KeyReconcile is the sync-bucket leader lease for the gateway quota reconcile.
-// It is deliberately distinct from vpcd's network-reconcile lease so the two
-// independent loops never share one mutex and block each other.
-const KeyReconcile = "quota.reconcile"
+// KVBucketQuotaReconcile is the leader-lock bucket for the gateway quota
+// reconcile. It is deliberately distinct from vpcd's network-reconcile lock so
+// the two independent loops never share one mutex and block each other.
+const KVBucketQuotaReconcile = "spinifex-quota-reconcile"
 
 // KVBucketAccountQuota holds the per-account limit overrides, keyed by
 // accountID. It is separate from the accounts bucket so a quota write never

spinifex/domains/admission/quota/seal.go

@@ -8,7 +8,7 @@ import (
 
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -73,7 +73,7 @@ func (s *Service) sealBucket(ctx context.Context) (jetstream.KeyValue, error) {
 		slog.ErrorContext(ctx, "Quota override seal has no JetStream client wired to open the override bucket")
 		return nil, errors.New(awserrors.ErrorInternalError)
 	}
-	kv, err := kvstore.GetOrCreateBucket(ctx, s.js, KVBucketAccountQuota, 1)
+	kv, err := kvutil.GetOrCreateBucket(ctx, s.js, KVBucketAccountQuota, 1)
 	if err != nil {
 		slog.ErrorContext(ctx, "Failed to open quota override bucket for seal", "err", err)
 		return nil, errors.New(awserrors.ErrorInternalError)
@@ -136,7 +136,7 @@ func (s *Service) SealAccountOverrides(ctx context.Context, accountID string) (*
 		switch {
 		case writeErr == nil:
 			return &OverrideSeal{Revision: newRev}, nil
-		case kvstore.IsRevisionConflict(writeErr):
+		case kvutil.IsRevisionConflict(writeErr):
 			continue
 		default:
 			slog.ErrorContext(ctx, "Failed to write sealed quota override record", "accountID", accountID, "err", writeErr)

spinifex/domains/dns/delegation.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	nsconfig "github.com/mulgadc/northstar/pkg/config"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go"
 
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 )
@@ -48,7 +48,7 @@ func (w *Writer) PlatformDelegation(name string) (servers []string, zone string,
 // Delegate writes, or with remove withdraws, an NS set in the base zone that
 // delegates each name to this cluster's nameservers. It takes the zone-writer
 // election first, so it never races the reconcile's own writes to the zone.
-func (w *Writer) Delegate(ctx context.Context, lease *kvsync.Lease, names []string, remove bool) (*ChangeResult, error) {
+func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, names []string, remove bool) (*ChangeResult, error) {
 	if !w.Enabled() {
 		return nil, errors.New("northstar is not configured on this node")
 	}
@@ -77,7 +77,7 @@ func (w *Writer) Delegate(ctx context.Context, lease *kvsync.Lease, names []stri
 	if len(batch.Changes) == 0 {
 		return &ChangeResult{}, nil
 	}
-	release, err := w.awaitWriterElection(ctx, lease)
+	release, err := w.awaitWriterElection(ctx, nc, holder)
 	if err != nil {
 		return nil, err
 	}
@@ -85,19 +85,19 @@ func (w *Writer) Delegate(ctx context.Context, lease *kvsync.Lease, names []stri
 	return w.ApplyBatch(batch)
 }
 
-func (w *Writer) awaitWriterElection(ctx context.Context, lease *kvsync.Lease) (func(), error) {
-	if lease == nil {
+func (w *Writer) awaitWriterElection(ctx context.Context, nc *nats.Conn, holder string) (func(), error) {
+	if nc == nil {
 		return func() {}, nil
 	}
 	tick := time.NewTicker(delegateLockPoll)
 	defer tick.Stop()
 	for {
-		if release, ok := reconcilelock.AcquireLeader(ctx, lease); ok {
+		if release, ok := reconcilelock.AcquireLeader(ctx, nc, KVBucketDNSReconcile, holder); ok {
 			return release, nil
 		}
 		select {
 		case <-ctx.Done():
-			return nil, fmt.Errorf("another writer has the zone: %w", ctx.Err())
+			return nil, fmt.Errorf("another node is writing the zone: %w", ctx.Err())
 		case <-tick.C:
 		}
 	}

spinifex/domains/dns/delegation_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, err := w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
+	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
 	require.NoError(t, err)
 
 	for _, name := range []string{"a.r53e2e.spx3.net.", "b.r53e2e.spx3.net"} {
@@ -29,7 +29,7 @@ func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 	assert.True(t, ok, "a name below the base domain is decided by the base zone")
 	assert.Empty(t, servers)
 
-	_, err = w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, true)
+	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
 	require.NoError(t, err)
 	servers, _, _, err = w.PlatformDelegation("a.r53e2e.spx3.net")
 	require.NoError(t, err)
@@ -43,11 +43,11 @@ func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 // servers written while the cluster published a larger set.
 func TestDelegateRemoveWithdrawsServersFromAnEarlierSet(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, err := w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, false)
+	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, false)
 	require.NoError(t, err)
 
 	w.nameservers = w.nameservers[:1]
-	_, err = w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, true)
+	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
 	require.NoError(t, err)
 	servers, _, _, err := w.PlatformDelegation("a.r53e2e.spx3.net")
 	require.NoError(t, err)
@@ -66,7 +66,7 @@ func TestPlatformDelegationIgnoresNamesOutsideTheBaseDomain(t *testing.T) {
 func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
 	for _, name := range []string{"ns1.spx3.net", "x.elb.spx3.net", "compute.spx3.net", "example.com", "spx3.net"} {
-		_, err := w.Delegate(t.Context(), nil, []string{"ok.spx3.net", name}, false)
+		_, err := w.Delegate(t.Context(), nil, "node1", []string{"ok.spx3.net", name}, false)
 		require.Error(t, err, name)
 	}
 	servers, _, _, err := w.PlatformDelegation("ok.spx3.net")
@@ -76,45 +76,22 @@ func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
 
 func TestDelegateWaitsForTheZoneWriterElection(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, _, js := testutil.StartTestJetStream(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
 	prev := delegateLockPoll
 	delegateLockPoll = 10 * time.Millisecond
 	t.Cleanup(func() { delegateLockPoll = prev })
 
-	release, ok := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "node2"))
+	release, ok := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "node2")
 	require.True(t, ok)
 	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
 	defer cancel()
-	node1 := dnsLease(js, "node1")
-	_, err := w.Delegate(ctx, node1, []string{"a.spx3.net"}, false)
-	require.ErrorContains(t, err, "another writer has the zone")
+	_, err := w.Delegate(ctx, nc, "node1", []string{"a.spx3.net"}, false)
+	require.ErrorContains(t, err, "another node is writing the zone")
 
 	release()
-	_, err = w.Delegate(t.Context(), node1, []string{"a.spx3.net"}, false)
+	_, err = w.Delegate(t.Context(), nc, "node1", []string{"a.spx3.net"}, false)
 	require.NoError(t, err)
 	servers, _, _, err := w.PlatformDelegation("a.spx3.net")
 	require.NoError(t, err)
 	assert.Len(t, servers, 2)
 }
-
-// The reconcile pass and a delegation on one node share the DNS lease, so a
-// delegation waits for this node's own pass as it would for another node's.
-func TestDelegateWaitsForThisNodesOwnReconcilePass(t *testing.T) {
-	w, _ := newMultiNodeTestWriter(t)
-	_, _, js := testutil.StartTestJetStream(t)
-	prev := delegateLockPoll
-	delegateLockPoll = 10 * time.Millisecond
-	t.Cleanup(func() { delegateLockPoll = prev })
-
-	shared := dnsLease(js, "node1")
-	release, ok := reconcilelock.AcquireLeader(t.Context(), shared)
-	require.True(t, ok, "the reconcile pass holds the lease")
-	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
-	defer cancel()
-	_, err := w.Delegate(ctx, shared, []string{"a.spx3.net"}, false)
-	require.ErrorContains(t, err, "another writer has the zone")
-
-	release()
-	_, err = w.Delegate(t.Context(), shared, []string{"a.spx3.net"}, false)
-	require.NoError(t, err)
-}

spinifex/domains/dns/reconcile.go

@@ -11,7 +11,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go"
 )
 
 // DefaultReconcileInterval is how often the drift backstop rebuilds managed
@@ -33,12 +33,6 @@ const DefaultReconcileInterval = reconciler.DefaultResync
 // costs a cycle for nothing.
 const DefaultComputeTimeout = 2 * time.Minute
 
-// NewReconcileLease returns the zone writer's election lease, shared by the
-// drift reconcile and Delegate so the two never write the zone together.
-func NewReconcileLease(store kvsync.KVStore, holder string) *kvsync.Lease {
-	return reconcilelock.NewLeaderLease(store, KeyReconcile, holder)
-}
-
 // DesiredFunc returns the full desired managed record set built from the live
 // resource inventory across all tenants. The daemon supplies it by enumerating
 // instances, load balancers, and EKS clusters.
@@ -108,8 +102,7 @@ type Reconciler struct {
 	// lands in. Distinct from both baseDomain and internalDomain, so pruning it
 	// needs its own read the same way internalDomain does for EC2.
 	serviceZone string
-	// lease elects the one node that writes the zones; nil skips the election.
-	lease *kvsync.Lease
+	nc          *nats.Conn
 	// writer applies the converged batch. The reconcile owns the zone object on
 	// the node that won the election, so the batch never leaves this process.
 	writer   *Writer
@@ -118,6 +111,7 @@ type Reconciler struct {
 	// computeTimeout bounds how long one pass spends building the desired set.
 	// A field rather than a constant so a test can shrink it.
 	computeTimeout time.Duration
+	holder         string
 	sources        []reconciler.Source
 }
 
@@ -127,15 +121,18 @@ type Reconciler struct {
 // changes should wake the loop. They are optional: with none supplied the loop
 // falls back to the interval alone, which is the behaviour that predates the
 // watch.
-func NewReconciler(cfg *config.Config, cluster *config.ClusterConfig, lease *kvsync.Lease, writer *Writer, desired DesiredFunc, sources ...reconciler.Source) *Reconciler {
+func NewReconciler(cfg *config.Config, cluster *config.ClusterConfig, nc *nats.Conn, writer *Writer, desired DesiredFunc, sources ...reconciler.Source) *Reconciler {
 	r := &Reconciler{
-		lease:          lease,
+		nc:             nc,
 		writer:         writer,
 		desired:        desired,
 		interval:       DefaultReconcileInterval,
 		computeTimeout: DefaultComputeTimeout,
 		sources:        sources,
 	}
+	if cfg != nil {
+		r.holder = cfg.Node
+	}
 	zoneCfg, ok := zoneS3Config(cfg)
 	if !ok || desired == nil || !writer.Enabled() {
 		return r
@@ -169,24 +166,25 @@ func (r *Reconciler) Run(ctx context.Context) {
 		// No revisit deadline: the desired set is a function of the buckets
 		// being watched, so a change is the only reason to run early.
 		Reconcile: func(ctx context.Context) (time.Duration, error) {
-			return r.reconcileOnce(ctx), nil
+			r.reconcileOnce(ctx)
+			return 0, nil
 		},
 		Resync: r.interval,
 	})
 }
 
 // reconcileOnce computes the converging batch and applies it, on whichever node
-// wins this cycle's election, and returns the election's revisit when it lost. The read that finds the stale records and the
+// wins this cycle's election. The read that finds the stale records and the
 // write that removes them now happen in one process, so a record re-added
 // between them is no longer deleted by a batch that predates it.
-func (r *Reconciler) reconcileOnce(ctx context.Context) time.Duration {
+func (r *Reconciler) reconcileOnce(ctx context.Context) {
 	if !r.enabled {
-		return 0
+		return
 	}
-	if r.lease != nil {
-		release, retry, elected := reconcilelock.ElectPass(ctx, r.lease)
+	if r.nc != nil {
+		release, elected := reconcilelock.AcquireLeader(ctx, r.nc, KVBucketDNSReconcile, r.holder)
 		if !elected {
-			return retry
+			return
 		}
 		defer release()
 	}
@@ -195,19 +193,18 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) time.Duration {
 		// A corrupt zone is handled in readZone; anything reaching here left the
 		// desired state unapplied, so surface it rather than burying it in WARN.
 		slog.Error("dns reconcile: compute batch failed, retrying next cycle", "error", err)
-		return 0
+		return
 	}
 	if len(batch) == 0 {
-		return 0
+		return
 	}
 	slog.Debug("dns reconcile: converging", "changes", len(batch))
 	res, err := r.writer.ApplyBatch(&ChangeBatch{Changes: batch})
 	if err != nil {
 		slog.Warn("dns reconcile: apply failed, retrying next cycle", "changes", len(batch), "error", err)
-		return 0
+		return
 	}
 	slog.Debug("dns reconcile: converged", "changes", res.Applied, "zones", res.Zones)
-	return 0
 }
 
 // computeBatchBounded runs computeBatch with a deadline, so a pass that never

spinifex/domains/dns/reconcile_election_test.go

@@ -8,50 +8,41 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/testkit"
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
-	"github.com/nats-io/nats.go/jetstream"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// electingReconcilerFor is reconcilerFor with a lease, so a pass runs the
-// election instead of skipping it.
-func electingReconcilerFor(w *Writer, lease *kvsync.Lease, desired func() DesiredSet) *Reconciler {
+// electingReconcilerFor is reconcilerFor with a live connection, so a pass runs
+// the election instead of skipping it.
+func electingReconcilerFor(w *Writer, nc *nats.Conn, holder string, desired func() DesiredSet) *Reconciler {
 	r := reconcilerFor(w, desired)
-	r.lease = lease
+	r.nc = nc
+	r.holder = holder
 	return r
 }
 
-// dnsLease returns holder's DNS lease on a store of its own, standing in for
-// a separate process.
-func dnsLease(js jetstream.JetStream, holder string) *kvsync.Lease {
-	return NewReconcileLease(kvstore.OpenSyncBucket(js), holder)
-}
-
 // The zone lock is gone, and what replaces it is that a zone has one writer:
 // the node holding the reconcile lease. A node that loses the election must
 // therefore not touch the object at all — if it did, two nodes could be
 // read-modify-writing the same zone with nothing serialising them.
 func TestReconcileOnce_ALoserOfTheElectionDoesNotWriteTheZone(t *testing.T) {
-	_, _, js := testutil.StartTestJetStream(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
 	w, objects := newTestWriter(t)
 	before := objects[testBase+".toml"]
 	require.NotEmpty(t, before)
 
 	// A peer holds the lease for the whole pass, so this node cannot win it.
-	release, elected := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "peer"))
+	release, elected := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "peer")
 	require.True(t, elected, "the peer has to hold the lease or this test proves nothing")
 
-	r := electingReconcilerFor(w, dnsLease(js, "node1"), func() DesiredSet {
+	r := electingReconcilerFor(w, nc, "node1", func() DesiredSet {
 		return DesiredSet{Changes: []Change{upsert("lb-1.elb."+testBase, "10.200.1.9")}}
 	})
-	revisit := r.reconcileOnce(t.Context())
+	r.reconcileOnce(t.Context())
 
 	assert.Equal(t, before, objects[testBase+".toml"],
 		"a node that did not win the election must leave the zone object exactly as it found it")
-	assert.Positive(t, revisit, "the loser asks to retry so a dead holder is taken over")
-	assert.LessOrEqual(t, revisit, reconcilelock.LeaderTTL/3, "the retry comes within a renewal interval")
 
 	release()
 }
@@ -59,10 +50,10 @@ func TestReconcileOnce_ALoserOfTheElectionDoesNotWriteTheZone(t *testing.T) {
 // The counterpart, so the test above is not passing because the pass was broken
 // for some other reason: with the lease free, the same reconciler writes.
 func TestReconcileOnce_TheWinnerOfTheElectionWritesTheZone(t *testing.T) {
-	_, _, js := testutil.StartTestJetStream(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
 	w, objects := newTestWriter(t)
 
-	r := electingReconcilerFor(w, dnsLease(js, "node1"), func() DesiredSet {
+	r := electingReconcilerFor(w, nc, "node1", func() DesiredSet {
 		return DesiredSet{Changes: []Change{upsert("lb-1.elb."+testBase, "10.200.1.9")}}
 	})
 	r.reconcileOnce(t.Context())

spinifex/domains/dns/reconcile_test.go

@@ -224,7 +224,7 @@ func TestReconciler_RunReturnsWhenDisabled(t *testing.T) {
 // path of Run: the startup pass must fire before any watch or resync activity,
 // and cancelling ctx must still let the loop return.
 func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
-	_, _, js := testutil.StartTestJetStream(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
 	endpoint, _ := fakeS3(t, "northstar")
 
 	var passes atomic.Int64
@@ -235,7 +235,8 @@ func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
 			Endpoint: endpoint, Bucket: "northstar", Region: "us-east-1",
 			AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET",
 		},
-		lease: dnsLease(js, "test-node"),
+		nc:     nc,
+		holder: "test-node",
 		desired: func() DesiredSet {
 			passes.Add(1)
 			return DesiredSet{}
@@ -267,7 +268,7 @@ func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
 // acquirable afterwards, not merely on the call returning, since it is the
 // leadership and not the pass that the rest of the cluster waits on.
 func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
-	_, _, js := testutil.StartTestJetStream(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
 	endpoint, _ := fakeS3(t, "northstar")
 
 	blocked := make(chan struct{})
@@ -280,7 +281,8 @@ func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
 			Endpoint: endpoint, Bucket: "northstar", Region: "us-east-1",
 			AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET",
 		},
-		lease:          dnsLease(js, "stuck-node"),
+		nc:             nc,
+		holder:         "stuck-node",
 		computeTimeout: 100 * time.Millisecond,
 		desired: func() DesiredSet {
 			<-blocked
@@ -300,7 +302,7 @@ func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
 		t.Fatal("reconcileOnce never returned: a stuck pass still holds the lease forever")
 	}
 
-	release, elected := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "next-node"))
+	release, elected := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "next-node")
 	require.True(t, elected, "another node must be able to take the next cycle")
 	release()
 }

spinifex/domains/dns/types.go

@@ -11,9 +11,9 @@ const (
 	// PUT so every northstar instance reloads just that zone immediately, instead
 	// of waiting for the next S3 sync poll. No queue group: all servers consume.
 	SubjectZoneReload = "dns.zone.reload"
-	// KeyReconcile elects the one node that writes the zones for a drift cycle.
-	// Its own key, so the election never blocks another reconcile loop.
-	KeyReconcile = "dns.reconcile"
+	// KVBucketDNSReconcile elects the one node that writes the zones for a drift
+	// cycle. Its own bucket, so the election never blocks another reconcile loop.
+	KVBucketDNSReconcile = "spinifex-dns-reconcile"
 	// PrivateZone is the fixed AWS-parity private DNS zone (IMDS synthHostname).
 	PrivateZone = "compute.internal"
 	// DefaultTTL is applied when a change omits a TTL.

spinifex/domains/ec2/account/seal.go

@@ -8,7 +8,7 @@ import (
 
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -109,7 +109,7 @@ func (s *AccountSettingsServiceImpl) SealAccount(ctx context.Context, accountID
 		switch {
 		case writeErr == nil:
 			return &AccountSettingsSeal{Revision: newRev}, nil
-		case kvstore.IsRevisionConflict(writeErr):
+		case kvutil.IsRevisionConflict(writeErr):
 			continue
 		default:
 			slog.ErrorContext(ctx, "Failed to write sealed account settings record", "accountID", accountID, "err", writeErr)

spinifex/domains/ec2/account/service_impl.go

@@ -13,7 +13,7 @@ import (
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -57,7 +57,7 @@ func NewAccountSettingsServiceImplWithNATS(ctx context.Context, cfg *config.Conf
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	settingsKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketAccountSettings, 10)
+	settingsKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketAccountSettings, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create account settings KV bucket: %w", err)
 	}
@@ -115,13 +115,13 @@ func mapSettingsError(ctx context.Context, accountID string, err error) error {
 		return errors.New(awserrors.ErrorBlocked)
 	case errors.Is(err, errSettingsContended):
 		slog.ErrorContext(ctx, "Account settings CAS retries exhausted under contention", "accountID", accountID)
-	case errors.Is(err, kvstore.ErrRead):
+	case errors.Is(err, kvutil.ErrRead):
 		slog.ErrorContext(ctx, "Failed to read account settings from KV", "accountID", accountID, "err", err)
-	case errors.Is(err, kvstore.ErrDecode):
+	case errors.Is(err, kvutil.ErrDecode):
 		slog.ErrorContext(ctx, "Corrupt account settings record in KV", "accountID", accountID, "err", err)
-	case errors.Is(err, kvstore.ErrEncode):
+	case errors.Is(err, kvutil.ErrEncode):
 		slog.ErrorContext(ctx, "Failed to marshal account settings record", "accountID", accountID, "err", err)
-	case errors.Is(err, kvstore.ErrWrite):
+	case errors.Is(err, kvutil.ErrWrite):
 		slog.ErrorContext(ctx, "Failed to write account settings to KV", "accountID", accountID, "err", err)
 	default:
 		slog.ErrorContext(ctx, "Account settings operation failed", "accountID", accountID, "err", err)
@@ -133,7 +133,7 @@ func mapSettingsError(ctx context.Context, accountID string, err error) error {
 // changed is always true, so every success is a leader-accepted write over an
 // active value; a seal landing between the read and the write fails the CAS.
 func (s *AccountSettingsServiceImpl) mutateSettings(ctx context.Context, accountID string, set func(*AccountSettingsRecord)) error {
-	_, err := kvstore.Update(ctx, s.settingsKV, settingsKey(accountID), kvstore.CASConfig{
+	_, err := kvutil.Update(ctx, s.settingsKV, settingsKey(accountID), kvutil.CASConfig{
 		CreateIfAbsent: true,
 		Exhausted:      func(string, int) error { return errSettingsContended },
 	}, func(rec *AccountSettingsRecord) (bool, error) {

spinifex/domains/ec2/ebs/metadata/metadata.go

@@ -110,23 +110,6 @@ type AMI struct {
 	Tags            map[string]string `json:"tags,omitempty"`
 	SourceDigest    *ImageDigest      `json:"source_digest,omitempty"`
 	State           string            `json:"state,omitempty"`
-	// UEFIData names the image's own UEFI variable store object; nil launches
-	// from the node's VARS template.
-	UEFIData *UEFIDataRef `json:"uefi_data,omitempty"`
-}
-
-// UEFIDataRef names an image's UEFI variable store object and the firmware it
-// was written under, so a launch can check the firmware without reading it.
-type UEFIDataRef struct {
-	Key      string       `json:"key"`
-	Firmware UEFIFirmware `json:"firmware"`
-}
-
-// UEFIFirmware is the firmware a variable store was written under.
-type UEFIFirmware struct {
-	Arch       string `json:"arch"`
-	VarsSize   int64  `json:"vars_size"`
-	CodeSHA256 string `json:"code_sha256"`
 }
 
 // ImageDigest identifies the artifact an AMI was imported from, not the

spinifex/domains/ec2/ebs/metadata/metadata_test.go

@@ -73,16 +73,6 @@ func TestAMIWithoutSourceDigestStillDecodes(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "ami-1", got.ImageID)
 	assert.Nil(t, got.SourceDigest)
-	assert.Nil(t, got.UEFIData, "a document with no uefi_data launches from the template")
-}
-
-// The UEFI store reference rides at schema version 2 under its JSON names.
-func TestAMIUEFIDataDecodesAtSchemaVersion2(t *testing.T) {
-	got, err := UnmarshalAMI([]byte(`{"schema_version":2,"image_id":"ami-1","uefi_data":` +
-		`{"key":"uefi/ami-1/k","firmware":{"arch":"x86_64","vars_size":540672,"code_sha256":"abc"}}}`))
-	require.NoError(t, err)
-	require.NotNil(t, got.UEFIData)
-	assert.Equal(t, UEFIDataRef{Key: "uefi/ami-1/k", Firmware: UEFIFirmware{Arch: "x86_64", VarsSize: 540672, CodeSHA256: "abc"}}, *got.UEFIData)
 }
 
 func TestMetadataRejectsUnknownSchemaVersion(t *testing.T) {

spinifex/domains/ec2/eigw/service_impl.go

@@ -19,7 +19,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -56,7 +56,7 @@ func NewEgressOnlyIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	eigwKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketEgressOnlyIGW, 10)
+	eigwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketEgressOnlyIGW, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketEgressOnlyIGW, err)
 	}
@@ -65,7 +65,7 @@ func NewEgressOnlyIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config
 	}
 
 	// Get or create VPC KV bucket for cross-resource ownership validation
-	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
@@ -90,7 +90,7 @@ func (s *EgressOnlyIGWServiceImpl) CreateEgressOnlyInternetGateway(ctx context.C
 		slog.ErrorContext(ctx, "VPC KV unavailable, cannot verify VPC ownership")
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, *input.VpcId)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, *input.VpcId)); err != nil {
 		slog.WarnContext(ctx, "CreateEgressOnlyInternetGateway: VPC not found for account", "vpcId", *input.VpcId, "accountID", accountID)
 		return nil, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", *input.VpcId)
 	}
@@ -109,7 +109,7 @@ func (s *EgressOnlyIGWServiceImpl) CreateEgressOnlyInternetGateway(ctx context.C
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal Egress-only IGW record: %w", err)
 	}
-	if _, err := s.eigwKV.Put(ctx, kvstore.AccountKey(accountID, eigwID), data); err != nil {
+	if _, err := s.eigwKV.Put(ctx, kvutil.AccountKey(accountID, eigwID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -127,7 +127,7 @@ func (s *EgressOnlyIGWServiceImpl) DeleteEgressOnlyInternetGateway(ctx context.C
 	}
 
 	eigwID := *input.EgressOnlyInternetGatewayId
-	key := kvstore.AccountKey(accountID, eigwID)
+	key := kvutil.AccountKey(accountID, eigwID)
 
 	// Verify the EIGW exists before deleting
 	if _, err := s.eigwKV.Get(ctx, key); err != nil {
@@ -178,7 +178,7 @@ func (s *EgressOnlyIGWServiceImpl) DescribeEgressOnlyInternetGateways(ctx contex
 	}
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey {
+		if key == kvutil.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {

spinifex/domains/ec2/eigw/service_impl_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -23,7 +23,7 @@ func setupTestEIGWService(t *testing.T) *EgressOnlyIGWServiceImpl {
 	// Create VPC KV bucket and register test VPCs so fail-closed ownership checks pass
 	vpcEntries := map[string][]byte{}
 	for _, vpcID := range []string{"vpc-test123", "vpc-tagged", "vpc-tagged2"} {
-		vpcEntries[kvstore.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
+		vpcEntries[kvutil.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
 	}
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, vpcEntries)
 
@@ -312,7 +312,7 @@ func TestCreateEgressOnlyInternetGateway_CrossAccountVPCRejected(t *testing.T) {
 	// Create VPC KV bucket with a VPC owned by testAccountID
 	vpcID := "vpc-alpha456"
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, map[string][]byte{
-		kvstore.AccountKey(testAccountID, vpcID): []byte(`{"vpc_id":"vpc-alpha456","state":"available"}`),
+		kvutil.AccountKey(testAccountID, vpcID): []byte(`{"vpc_id":"vpc-alpha456","state":"available"}`),
 	})
 
 	var err error

spinifex/domains/ec2/eip/allocate_idempotency.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -36,7 +36,7 @@ func (s *EIPServiceImpl) allocateOnce(ctx context.Context, accountID string, all
 	if key == "" || s.idemKV == nil {
 		return alloc()
 	}
-	kvKey := kvstore.AccountKey(accountID, idempotencyKeyHash(key))
+	kvKey := kvutil.AccountKey(accountID, idempotencyKeyHash(key))
 
 	if out := s.cachedAllocation(ctx, kvKey); out != nil {
 		slog.WarnContext(ctx, "AllocateAddress: returning first result for a retried request",

spinifex/domains/ec2/eip/lease_rebind.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/domains/network/projection"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/topology"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -80,7 +80,7 @@ func (s *EIPServiceImpl) AllocationExists(ctx context.Context, allocationID stri
 	}
 	suffix := "." + allocationID
 	for _, k := range keys {
-		if k != kvstore.VersionKey && strings.HasSuffix(k, suffix) {
+		if k != kvutil.VersionKey && strings.HasSuffix(k, suffix) {
 			return true, nil
 		}
 	}
@@ -98,7 +98,7 @@ func (s *EIPServiceImpl) findByAllocationIDAnyAccount(ctx context.Context, alloc
 
 	suffix := "." + allocationID
 	for _, k := range keys {
-		if k == kvstore.VersionKey || !strings.HasSuffix(k, suffix) {
+		if k == kvutil.VersionKey || !strings.HasSuffix(k, suffix) {
 			continue
 		}
 		entry, err := s.eipKV.Get(ctx, k)

spinifex/domains/ec2/eip/service_impl.go

@@ -20,7 +20,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -66,7 +66,7 @@ func NewEIPServiceImpl(ctx context.Context, natsConn *nats.Conn, externalIPAM *e
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	eipKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketEIPs, 10)
+	eipKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketEIPs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketEIPs, err)
 	}
@@ -76,7 +76,7 @@ func NewEIPServiceImpl(ctx context.Context, natsConn *nats.Conn, externalIPAM *e
 
 	// TTL'd, so a failure to create it is not worth refusing to serve EIPs over:
 	// a nil bucket only costs post-completion dedupe.
-	idemKV, err := kvstore.GetOrCreateBucketWithTTL(ctx, js, KVBucketEIPIdempotency, 1, eipIdempotencyTTL)
+	idemKV, err := kvutil.GetOrCreateBucketWithTTL(ctx, js, KVBucketEIPIdempotency, 1, eipIdempotencyTTL)
 	if err != nil {
 		slog.Warn("EIP service: idempotency bucket unavailable; a retried AllocateAddress may allocate twice",
 			"bucket", KVBucketEIPIdempotency, "err", err)
@@ -146,7 +146,7 @@ func (s *EIPServiceImpl) allocateAddress(ctx context.Context, input *ec2.Allocat
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal EIP record: %w", err)
 	}
-	if _, err := s.eipKV.Put(ctx, kvstore.AccountKey(accountID, allocID), data); err != nil {
+	if _, err := s.eipKV.Put(ctx, kvutil.AccountKey(accountID, allocID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -167,7 +167,7 @@ func (s *EIPServiceImpl) ReleaseAddress(ctx context.Context, input *ec2.ReleaseA
 	}
 
 	allocID := *input.AllocationId
-	key := kvstore.AccountKey(accountID, allocID)
+	key := kvutil.AccountKey(accountID, allocID)
 
 	entry, err := s.eipKV.Get(ctx, key)
 	if err != nil {
@@ -206,7 +206,7 @@ func (s *EIPServiceImpl) AssociateAddress(ctx context.Context, input *ec2.Associ
 	}
 
 	allocID := *input.AllocationId
-	key := kvstore.AccountKey(accountID, allocID)
+	key := kvutil.AccountKey(accountID, allocID)
 
 	entry, err := s.eipKV.Get(ctx, key)
 	if err != nil {
@@ -536,7 +536,7 @@ func (s *EIPServiceImpl) findByENI(ctx context.Context, accountID, eniID string)
 		if err := ctx.Err(); err != nil {
 			return nil, err
 		}
-		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.eipKV.Get(ctx, k)
@@ -590,7 +590,7 @@ func (s *EIPServiceImpl) DescribeAddresses(ctx context.Context, input *ec2.Descr
 
 	var addresses []*ec2.Address
 	for _, k := range keys {
-		if k == kvstore.VersionKey {
+		if k == kvutil.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -699,7 +699,7 @@ func (s *EIPServiceImpl) DescribeAddressesAttribute(ctx context.Context, input *
 			if id == nil {
 				continue
 			}
-			key := kvstore.AccountKey(accountID, *id)
+			key := kvutil.AccountKey(accountID, *id)
 			entry, err := s.eipKV.Get(ctx, key)
 			if err != nil {
 				continue // not found, skip
@@ -722,7 +722,7 @@ func (s *EIPServiceImpl) DescribeAddressesAttribute(ctx context.Context, input *
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
 		for _, k := range keys {
-			if k == kvstore.VersionKey {
+			if k == kvutil.VersionKey {
 				continue
 			}
 			if !strings.HasPrefix(k, prefix) {
@@ -819,7 +819,7 @@ func (s *EIPServiceImpl) findByAssociationID(ctx context.Context, accountID, ass
 	}
 
 	for _, k := range keys {
-		if k == kvstore.VersionKey {
+		if k == kvutil.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -857,7 +857,7 @@ func (s *EIPServiceImpl) AssociatedPublicIPForInstance(ctx context.Context, acco
 		return "", false
 	}
 	for _, k := range keys {
-		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.eipKV.Get(ctx, k)

spinifex/domains/ec2/igw/service_impl.go

@@ -22,7 +22,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -93,7 +93,7 @@ func NewIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	igwKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketIGW, 10)
+	igwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketIGW, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketIGW, err)
 	}
@@ -102,13 +102,13 @@ func NewIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 	}
 
 	// Get or create VPC KV bucket for cross-resource ownership validation
-	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
 
 	// Read EIP associations so detach can refuse while addresses are mapped.
-	eipKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
+	eipKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get EIP KV bucket: %w", err)
 	}
@@ -142,7 +142,7 @@ func (s *IGWServiceImpl) createIGW(ctx context.Context, input *ec2.CreateInterne
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal IGW record: %w", err)
 	}
-	if _, err := s.igwKV.Put(ctx, kvstore.AccountKey(accountID, igwID), data); err != nil {
+	if _, err := s.igwKV.Put(ctx, kvutil.AccountKey(accountID, igwID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -160,7 +160,7 @@ func (s *IGWServiceImpl) DeleteInternetGateway(ctx context.Context, input *ec2.D
 	}
 
 	igwID := *input.InternetGatewayId
-	key := kvstore.AccountKey(accountID, igwID)
+	key := kvutil.AccountKey(accountID, igwID)
 
 	entry, err := s.igwKV.Get(ctx, key)
 	if err != nil {
@@ -236,7 +236,7 @@ func (s *IGWServiceImpl) DescribeInternetGateways(ctx context.Context, input *ec
 	foundIDs := make(map[string]bool)
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey {
+		if key == kvutil.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -326,7 +326,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A
 
 	igwID := *input.InternetGatewayId
 	vpcID := *input.VpcId
-	key := kvstore.AccountKey(accountID, igwID)
+	key := kvutil.AccountKey(accountID, igwID)
 
 	entry, err := s.igwKV.Get(ctx, key)
 	if err != nil {
@@ -347,7 +347,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A
 		slog.ErrorContext(ctx, "VPC KV unavailable, cannot verify VPC ownership")
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID)); err != nil {
 		slog.WarnContext(ctx, "AttachInternetGateway: VPC not found for account", "vpcId", vpcID, "accountID", accountID)
 		return nil, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
 	}
@@ -388,7 +388,7 @@ func (s *IGWServiceImpl) DetachInternetGateway(ctx context.Context, input *ec2.D
 
 	igwID := *input.InternetGatewayId
 	vpcID := *input.VpcId
-	key := kvstore.AccountKey(accountID, igwID)
+	key := kvutil.AccountKey(accountID, igwID)
 
 	entry, err := s.igwKV.Get(ctx, key)
 	if err != nil {
@@ -449,7 +449,7 @@ func (s *IGWServiceImpl) DetachInternetGateway(ctx context.Context, input *ec2.D
 // associated into vpcID, whether to an interface or a NAT gateway. The listing
 // is filtered server-side to the account's keys.
 func (s *IGWServiceImpl) vpcHasMappedAddress(ctx context.Context, accountID, vpcID string) (bool, error) {
-	lister, err := s.eipKV.ListKeysFiltered(ctx, kvstore.AccountKey(accountID, "*"))
+	lister, err := s.eipKV.ListKeysFiltered(ctx, kvutil.AccountKey(accountID, "*"))
 	if err != nil {
 		return false, err
 	}
@@ -493,11 +493,11 @@ func (s *IGWServiceImpl) CreateAttachedInternetGateway(ctx context.Context, acco
 	if s.vpcKV == nil {
 		return false, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID)); err != nil {
 		return false, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
 	}
 
-	key := kvstore.AccountKey(accountID, igwID)
+	key := kvutil.AccountKey(accountID, igwID)
 	data, err := json.Marshal(IGWRecord{
 		InternetGatewayId: igwID,
 		VpcId:             vpcID,
@@ -558,7 +558,7 @@ func (s *IGWServiceImpl) AttachmentIntent(ctx context.Context, accountID, vpcID
 	}
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.igwKV.Get(ctx, key)
@@ -599,7 +599,7 @@ func (s *IGWServiceImpl) AttachmentIntent(ctx context.Context, accountID, vpcID
 // Writing only on the pending transition keeps a pass from waking the drift loop.
 func MarkAttached(ctx context.Context, kv jetstream.KeyValue, recordKey, vpcID string) error {
 	confirmed := false
-	record, err := kvstore.Update(ctx, kv, recordKey, kvstore.CASConfig{}, func(r *IGWRecord) (bool, error) {
+	record, err := kvutil.Update(ctx, kv, recordKey, kvutil.CASConfig{}, func(r *IGWRecord) (bool, error) {
 		// Reset per attempt: a retried CAS may see a record another writer has
 		// already moved out of pending.
 		confirmed = r.VpcId == vpcID && r.AttachState == AttachStatePending

spinifex/domains/ec2/igw/service_impl_test.go

@@ -13,7 +13,7 @@ import (
 	ec2eip "github.com/mulgadc/spinifex/spinifex/domains/ec2/eip"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -55,7 +55,7 @@ func setupTestIGWService(t *testing.T) (*IGWServiceImpl, *nats.Conn) {
 	// Create VPC KV bucket and register test VPCs so fail-closed ownership checks pass
 	vpcEntries := map[string][]byte{}
 	for _, vpcID := range []string{"vpc-test123", "vpc-other", "vpc-lifecycle", "vpc-event-test"} {
-		vpcEntries[kvstore.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
+		vpcEntries[kvutil.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
 	}
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, vpcEntries)
 
@@ -68,7 +68,7 @@ func setupTestIGWService(t *testing.T) (*IGWServiceImpl, *nats.Conn) {
 // gateway come up. Without it an attach stays pending and reports no attachment.
 func confirmAttach(t *testing.T, svc *IGWServiceImpl, igwID, vpcID string) {
 	t.Helper()
-	require.NoError(t, MarkAttached(context.Background(), svc.igwKV, kvstore.AccountKey(testAccountID, igwID), vpcID))
+	require.NoError(t, MarkAttached(context.Background(), svc.igwKV, kvutil.AccountKey(testAccountID, igwID), vpcID))
 }
 
 func createTestIGW(t *testing.T, svc *IGWServiceImpl) string {
@@ -331,7 +331,7 @@ func TestDescribeInternetGateways_LegacyRecordReportsAttachment(t *testing.T) {
 	svc, _ := setupTestIGWService(t)
 	igwID := createTestIGW(t, svc)
 
-	key := kvstore.AccountKey(testAccountID, igwID)
+	key := kvutil.AccountKey(testAccountID, igwID)
 	entry, err := svc.igwKV.Get(context.Background(), key)
 	require.NoError(t, err)
 	var record IGWRecord
@@ -398,7 +398,7 @@ func TestMarkAttached_NoRewriteWhenAlreadyAttached(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 
-	key := kvstore.AccountKey(testAccountID, igwID)
+	key := kvutil.AccountKey(testAccountID, igwID)
 	confirmAttach(t, svc, igwID, "vpc-test123")
 	first, err := svc.igwKV.Get(context.Background(), key)
 	require.NoError(t, err)
@@ -605,7 +605,7 @@ func TestDetachInternetGateway_MappedPublicAddress(t *testing.T) {
 	putEIP := func(rec ec2eip.EIPRecord) {
 		data, err := json.Marshal(rec)
 		require.NoError(t, err)
-		_, err = svc.eipKV.Put(context.Background(), kvstore.AccountKey(testAccountID, rec.AllocationId), data)
+		_, err = svc.eipKV.Put(context.Background(), kvutil.AccountKey(testAccountID, rec.AllocationId), data)
 		require.NoError(t, err)
 	}
 	putEIP(ec2eip.EIPRecord{AllocationId: "eipalloc-other", AssociationId: "eipassoc-other", VpcId: "vpc-other", State: "associated"})
@@ -614,7 +614,7 @@ func TestDetachInternetGateway_MappedPublicAddress(t *testing.T) {
 	// Another account's address naming the same VPC ID is not this account's dependency.
 	foreign, err := json.Marshal(ec2eip.EIPRecord{AllocationId: "eipalloc-foreign", AssociationId: "eipassoc-foreign", VpcId: "vpc-test123", State: "associated"})
 	require.NoError(t, err)
-	_, err = svc.eipKV.Put(context.Background(), kvstore.AccountKey("999999999999", "eipalloc-foreign"), foreign)
+	_, err = svc.eipKV.Put(context.Background(), kvutil.AccountKey("999999999999", "eipalloc-foreign"), foreign)
 	require.NoError(t, err)
 
 	detach := func() error {
@@ -828,7 +828,7 @@ func TestAttachInternetGateway_CrossAccountVPCRejected(t *testing.T) {
 	require.NoError(t, err)
 
 	vpcID := "vpc-alpha123"
-	_, err = vpcKV.Put(t.Context(), kvstore.AccountKey(testAccountID, vpcID), []byte(`{"vpc_id":"vpc-alpha123","state":"available"}`))
+	_, err = vpcKV.Put(t.Context(), kvutil.AccountKey(testAccountID, vpcID), []byte(`{"vpc_id":"vpc-alpha123","state":"available"}`))
 	require.NoError(t, err)
 
 	// Refresh service to pick up the VPC KV bucket

spinifex/domains/ec2/image/service_impl.go

@@ -18,7 +18,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	ec2platform "github.com/mulgadc/spinifex/spinifex/domains/ec2/platform"
 	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -40,9 +39,6 @@ type CreateImageParams struct {
 	RootVolumeID  string
 	SourceImageID string
 	IsRunning     bool // logged only: the provider's CreateSnapshot drains a live volume itself
-	// UEFI is set for an instance that boots through UEFI, whose variable
-	// store the image carries.
-	UEFI bool
 }
 
 // ImageServiceImpl handles AMI image operations with S3 storage.
@@ -53,9 +49,6 @@ type ImageServiceImpl struct {
 	natsConn   *nats.Conn
 	metadata   *ebsmetadata.Store
 	provider   ebsprovider.EBSProvider
-
-	uefiRecords UEFIStoreRecords
-	nodeID      string
 }
 
 // SetEBSProvider injects the provider boundary used by the control plane.
@@ -451,23 +444,12 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		"rootVolumeId", params.RootVolumeID, "amiId", amiID, "snapshotId", snapshotID,
 		"isRunning", params.IsRunning)
 
-	// Step 1: Copy the instance's UEFI variable store to the image's own key,
-	// before the snapshot, so a refusal here leaves no snapshot behind.
-	var uefiData *ebsmetadata.UEFIDataRef
-	if params.UEFI {
-		var err error
-		uefiData, err = s.copyInstanceUEFIStore(context.Background(), *input.InstanceId, amiID)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Step 2: Snapshot root volume
+	// Step 1: Snapshot root volume
 	if err := s.snapshotVolume(params.RootVolumeID, snapshotID, accountID); err != nil {
 		return nil, err
 	}
 
-	// Step 3: Read source volume config for size
+	// Step 2: Read source volume config for size
 	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, params.RootVolumeID)
 	if err != nil {
 		slog.Error("CreateImageFromInstance: failed to read volume metadata", "volumeId", params.RootVolumeID, "err", err)
@@ -475,7 +457,7 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 	}
 	volumeSizeGiB := volMeta.CapacityGiB
 
-	// Step 4: Read source AMI config for architecture, platform, etc.
+	// Step 3: Read source AMI config for architecture, platform, etc.
 	sourceAMI := ebsmetadata.AMI{
 		Architecture:    "x86_64",
 		PlatformDetails: "Linux/UNIX",
@@ -490,13 +472,13 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		sourceAMI = srcCfg
 	}
 
-	// Step 5: Store snapshot metadata
+	// Step 4: Store snapshot metadata
 	if err := s.putSnapshotMetadata(context.Background(), snapshotID, params.RootVolumeID, volumeSizeGiB, accountID); err != nil {
 		slog.Error("CreateImageFromInstance: failed to write snapshot metadata", "snapshotId", snapshotID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// Step 6: Build and store AMI config
+	// Step 5: Build and store AMI config
 	meta := ebsmetadata.AMI{
 		ImageID:         amiID,
 		Name:            name,
@@ -515,12 +497,10 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		Distro:       sourceAMI.Distro,
 		DistroFamily: sourceAMI.DistroFamily,
 		// Snapshot succeeded before this point, so the image is complete.
-		State:    "available",
-		UEFIData: uefiData,
+		State: "available",
 	}
 
 	if err := s.putAMIConfig(context.Background(), amiID, meta); err != nil {
-		// A UEFI object left under the image's prefix is in the orphan report.
 		slog.Error("CreateImageFromInstance: failed to store AMI config", "amiId", amiID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
@@ -876,17 +856,6 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 
 	tags := mergeCopyImageTags(srcMeta.Tags, input.TagSpecifications, aws.BoolValue(input.CopyImageTags))
 
-	uefiData, err := s.copyImageUEFIStore(ctx, srcMeta, newImageID)
-	if err != nil {
-		slog.ErrorContext(ctx, "CopyImage: failed to copy the UEFI variable store",
-			"sourceImageId", sourceImageID, "amiId", newImageID, "orphanSnapshotId", newSnapshotID, "err", err)
-		if delErr := s.metadata.DeleteSnapshot(ctx, accountID, newSnapshotID); delErr != nil {
-			slog.ErrorContext(ctx, "CopyImage: failed to roll back orphaned snapshot metadata",
-				"snapshotId", newSnapshotID, "err", delErr)
-		}
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-
 	meta := ebsmetadata.AMI{
 		ImageID:         newImageID,
 		Name:            name,
@@ -907,18 +876,12 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 		SourceDigest:    srcMeta.SourceDigest,
 		// Zero-copy: the new config shares the source's already-durable
 		// snapshot, so the image is complete as soon as this is written.
-		State:    "available",
-		UEFIData: uefiData,
+		State: "available",
 	}
 
 	if err := s.putAMIConfig(ctx, newImageID, meta); err != nil {
-		var orphanUEFIKey string
-		if uefiData != nil {
-			orphanUEFIKey = uefiData.Key
-		}
-		// The copied UEFI object stays under the new image's prefix; the log names it.
 		slog.ErrorContext(ctx, "CopyImage: failed to write AMI config",
-			"amiId", newImageID, "orphanSnapshotId", newSnapshotID, "orphanUEFIKey", orphanUEFIKey, "err", err)
+			"amiId", newImageID, "orphanSnapshotId", newSnapshotID, "err", err)
 		// Best-effort rollback of the orphaned snapshot metadata.
 		if delErr := s.metadata.DeleteSnapshot(ctx, accountID, newSnapshotID); delErr != nil {
 			slog.ErrorContext(ctx, "CopyImage: failed to roll back orphaned snapshot metadata",
@@ -1162,10 +1125,10 @@ func pickRootSnapshotBDM(mappings []*ec2.BlockDeviceMapping, rootDeviceName *str
 	return nil
 }
 
-// DeregisterImage hard-deletes the image's UEFI store prefix, then the AMI
-// document, so a retry after a failed delete finishes the job. The legacy
-// {imageID}/config.json goes too, or the metadata store's read fallback would
-// resurrect the image. The backing snapshot is untouched, as on AWS.
+// DeregisterImage hard-deletes the AMI document. The pre-provider
+// {imageID}/config.json goes too, or the metadata store's legacy read fallback
+// would resurrect the image. Backing snapshot is untouched, so operators run
+// delete-snapshot separately to reclaim block storage.
 func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.DeregisterImageInput, accountID string) (*ec2.DeregisterImageOutput, error) {
 	if input == nil || input.ImageId == nil || *input.ImageId == "" {
 		return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -1177,12 +1140,6 @@ func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.Dereg
 		return nil, err
 	}
 
-	// Only the image's own prefix: a copy or a source instance has its own.
-	if n, err := uefistore.DeleteObjects(ctx, s.store, s.bucketName, imageID); err != nil {
-		slog.ErrorContext(ctx, "DeregisterImage: failed to delete the UEFI variable store", "imageId", imageID, "deleted", n, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-
 	if err := s.metadata.DeleteAMI(ctx, imageID); err != nil && !objectstore.IsNoSuchKeyError(err) {
 		slog.ErrorContext(ctx, "DeregisterImage: failed to delete AMI document", "imageId", imageID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)

spinifex/domains/ec2/image/uefi.go

@@ -1,101 +0,0 @@
-package image
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-
-	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-)
-
-// uefiPointerAttempts bounds how often CreateImage re-reads a running
-// instance's pointer after the object it named was pruned behind a newer one.
-const uefiPointerAttempts = 3
-
-// UEFIStoreRecords is the store-record read CreateImage makes of its source
-// instance, answered by the stream leader so it sees the last landed pointer.
-type UEFIStoreRecords interface {
-	LeaderGet(ctx context.Context, instanceID string) (*uefistore.UEFIStoreRecord, uint64, error)
-}
-
-var _ UEFIStoreRecords = (*uefistore.Store)(nil)
-
-// SetUEFIStores wires the store records CreateImage reads a UEFI instance's
-// pointer from, and the node ID it writes image store objects as.
-func (s *ImageServiceImpl) SetUEFIStores(records UEFIStoreRecords, nodeID string) {
-	s.uefiRecords = records
-	s.nodeID = nodeID
-}
-
-// copyInstanceUEFIStore copies the object instanceID's store record points at
-// to a fresh key under imageID and returns its reference, or nil when the
-// instance has no record or no pointer. The error is an AWS error code.
-func (s *ImageServiceImpl) copyInstanceUEFIStore(ctx context.Context, instanceID, imageID string) (*ebsmetadata.UEFIDataRef, error) {
-	if s.uefiRecords == nil || s.nodeID == "" {
-		slog.ErrorContext(ctx, "CreateImage: no UEFI store records are wired on this node", "instanceId", instanceID)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-	// Once a live pointer is seen, a record that then vanishes is a terminate
-	// racing this request, not an instance without a store.
-	sawPointer := false
-	for range uefiPointerAttempts {
-		rec, _, err := s.uefiRecords.LeaderGet(ctx, instanceID)
-		switch {
-		case errors.Is(err, kvstore.ErrNotFound) && sawPointer:
-			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
-		case errors.Is(err, kvstore.ErrNotFound):
-			return nil, nil
-		case err != nil:
-			slog.ErrorContext(ctx, "CreateImage: failed to read the UEFI store record", "instanceId", instanceID, "err", err)
-			return nil, errors.New(awserrors.ErrorServerInternal)
-		case rec.State != uefistore.StateLive:
-			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
-		case rec.Pointer == nil:
-			return nil, nil
-		}
-		sawPointer = true
-		ptr := rec.Pointer
-		key, err := uefistore.CopyObject(ctx, s.store, s.bucketName, ptr.Key, uefistore.ObjectMeta{
-			OwnerID: imageID, WriterNode: s.nodeID, Firmware: ptr.Firmware, Source: uefistore.SourceInstance(instanceID),
-		})
-		if errors.Is(err, uefistore.ErrObjectNotFound) {
-			// A write-back moved the pointer and a prune removed this one.
-			continue
-		}
-		if err != nil {
-			slog.ErrorContext(ctx, "CreateImage: failed to copy the UEFI variable store",
-				"instanceId", instanceID, "imageId", imageID, "key", ptr.Key, "err", err)
-			return nil, errors.New(awserrors.ErrorServerInternal)
-		}
-		slog.InfoContext(ctx, "CreateImage: copied the UEFI variable store",
-			"instanceId", instanceID, "imageId", imageID, "sourceKey", ptr.Key, "key", key)
-		return uefistore.AMIRef(key, ptr.Firmware), nil
-	}
-	slog.ErrorContext(ctx, "CreateImage: the UEFI store pointer kept naming a missing object",
-		"instanceId", instanceID, "attempts", uefiPointerAttempts)
-	return nil, errors.New(awserrors.ErrorServerInternal)
-}
-
-// copyImageUEFIStore copies the store src names to a fresh key under imageID,
-// so deregistering either image leaves the other's intact. It returns nil
-// when src names none.
-func (s *ImageServiceImpl) copyImageUEFIStore(ctx context.Context, src ebsmetadata.AMI, imageID string) (*ebsmetadata.UEFIDataRef, error) {
-	store, err := uefistore.ImageVarStoreFromAMI(src)
-	if err != nil || store == nil {
-		return nil, err
-	}
-	if s.nodeID == "" {
-		return nil, errors.New("no node ID is wired to write UEFI store objects as")
-	}
-	key, err := uefistore.CopyObject(ctx, s.store, s.bucketName, store.Key, uefistore.ObjectMeta{
-		OwnerID: imageID, WriterNode: s.nodeID, Firmware: store.Firmware, Source: uefistore.SourceImage(src.ImageID),
-	})
-	if err != nil {
-		return nil, fmt.Errorf("copy UEFI store %s to %s: %w", store.Key, imageID, err)
-	}
-	return uefistore.AMIRef(key, store.Firmware), nil
-}

spinifex/domains/ec2/image/uefi_test.go

@@ -1,417 +0,0 @@
-package image
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"io"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	awss3 "github.com/aws/aws-sdk-go/service/s3"
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	uefiTestInstance = "i-0123456789abcdef0"
-	uefiTestVarsSize = 4096
-)
-
-var uefiTestFirmware = uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: uefiTestVarsSize, CodeSHA256: "code-sha"}
-
-func uefiRegion(tag string) []byte {
-	b := make([]byte, uefiTestVarsSize)
-	copy(b, tag)
-	return b
-}
-
-// uefiImageFixture is an image service wired to a real store-record bucket,
-// with a UEFI instance whose root volume CreateImage can snapshot.
-type uefiImageFixture struct {
-	svc       *ImageServiceImpl
-	snapshots *countingSnapshots
-	objects   *objectstore.MemoryObjectStore
-	records   *uefistore.Store
-	volume    ebsmetadata.Volume
-}
-
-func newUEFIImageFixture(t *testing.T) *uefiImageFixture {
-	t.Helper()
-	svc, objects, provider := setupProviderSnapshotImageService(t)
-	_, _, js := testutil.StartTestJetStream(t)
-	records := uefistore.NewStore(js)
-	require.NoError(t, records.Open(t.Context()))
-	svc.SetUEFIStores(records, "node-a")
-	volume := seedProviderVolume(t, svc, provider, "vol-uefi001", 8)
-	snapshots := &countingSnapshots{EBSProvider: provider}
-	svc.SetEBSProvider(snapshots)
-	return &uefiImageFixture{svc: svc, snapshots: snapshots, objects: objects, records: records, volume: volume}
-}
-
-// countingSnapshots counts the snapshots CreateImage asks the provider for.
-type countingSnapshots struct {
-	ebsprovider.EBSProvider
-
-	created int
-}
-
-func (c *countingSnapshots) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
-	c.created++
-	return c.EBSProvider.CreateSnapshot(ctx, req)
-}
-
-// assertNoSnapshot checks a refused CreateImage took no snapshot and wrote no
-// snapshot document.
-func (f *uefiImageFixture) assertNoSnapshot(t *testing.T) {
-	t.Helper()
-	assert.Zero(t, f.snapshots.created, "a refused CreateImage takes no snapshot")
-	snaps, err := f.svc.MetadataStore().ListAllSnapshots(t.Context())
-	require.NoError(t, err)
-	assert.Empty(t, snaps, "a refused CreateImage writes no snapshot document")
-}
-
-// putInstanceStore puts vars at a fresh key under the instance and points a
-// live store record at it.
-func (f *uefiImageFixture) putInstanceStore(t *testing.T, vars []byte) string {
-	t.Helper()
-	key := f.putObject(t, uefiTestInstance, vars)
-	_, err := f.records.Create(t.Context(), &uefistore.UEFIStoreRecord{
-		InstanceID: uefiTestInstance, AccountID: testAccountID, State: uefistore.StateLive, Generation: 1,
-		Pointer: &uefistore.UEFIVarStoreRef{Key: key, Firmware: uefiTestFirmware, WriterNode: "node-a", WrittenAt: time.Now().UTC()},
-	})
-	require.NoError(t, err)
-	return key
-}
-
-func (f *uefiImageFixture) putObject(t *testing.T, owner string, vars []byte) string {
-	t.Helper()
-	key, err := uefistore.NewObjectKey(owner)
-	require.NoError(t, err)
-	require.NoError(t, uefistore.PutObject(t.Context(), f.objects, testBucket, key, vars, uefistore.ObjectMeta{
-		OwnerID: owner, WriterNode: "node-a", Firmware: uefiTestFirmware, Source: uefistore.SourceWriteBack,
-	}))
-	return key
-}
-
-func (f *uefiImageFixture) createImage(t *testing.T, uefi bool) (string, error) {
-	t.Helper()
-	out, err := f.svc.CreateImageFromInstance(CreateImageParams{
-		Input:        &ec2.CreateImageInput{InstanceId: aws.String(uefiTestInstance), Name: aws.String("uefi-image")},
-		RootVolumeID: f.volume.VolumeID,
-		UEFI:         uefi,
-	}, testAccountID)
-	if err != nil {
-		return "", err
-	}
-	return aws.StringValue(out.ImageId), nil
-}
-
-func (f *uefiImageFixture) ami(t *testing.T, imageID string) ebsmetadata.AMI {
-	t.Helper()
-	ami, err := f.svc.MetadataStore().GetAMI(t.Context(), imageID)
-	require.NoError(t, err)
-	return ami
-}
-
-func (f *uefiImageFixture) region(t *testing.T, key string) []byte {
-	t.Helper()
-	out, err := f.objects.GetObject(t.Context(), &awss3.GetObjectInput{Bucket: aws.String(testBucket), Key: aws.String(key)})
-	require.NoError(t, err)
-	defer out.Body.Close()
-	zr, err := gzip.NewReader(out.Body)
-	require.NoError(t, err)
-	b, err := io.ReadAll(zr)
-	require.NoError(t, err)
-	return b
-}
-
-func (f *uefiImageFixture) keys(t *testing.T, owner string) []string {
-	t.Helper()
-	keys, err := uefistore.ListObjects(t.Context(), f.objects, testBucket, owner)
-	require.NoError(t, err)
-	return keys
-}
-
-func (f *uefiImageFixture) copyImage(t *testing.T, src, name string) string {
-	t.Helper()
-	out, err := f.svc.CopyImage(t.Context(), validCopyImageServiceInput(src, name), testAccountID)
-	require.NoError(t, err)
-	return aws.StringValue(out.ImageId)
-}
-
-func (f *uefiImageFixture) deregister(t *testing.T, imageID string) error {
-	t.Helper()
-	_, err := f.svc.DeregisterImage(t.Context(), &ec2.DeregisterImageInput{ImageId: aws.String(imageID)}, testAccountID)
-	return err
-}
-
-// CreateImage copies the object the instance's record points at to the
-// image's own prefix, and the AMI document names it with its firmware.
-func TestCreateImage_CopiesTheInstancesUEFIStore(t *testing.T) {
-	f := newUEFIImageFixture(t)
-	instanceKey := f.putInstanceStore(t, uefiRegion("guest variables"))
-
-	imageID, err := f.createImage(t, true)
-	require.NoError(t, err)
-
-	ami := f.ami(t, imageID)
-	require.NotNil(t, ami.UEFIData)
-	assert.Equal(t, []string{ami.UEFIData.Key}, f.keys(t, imageID), "the image has one object under its own prefix")
-	assert.Equal(t, uefiRegion("guest variables"), f.region(t, ami.UEFIData.Key))
-	assert.Equal(t, uefistore.AMIRef(ami.UEFIData.Key, uefiTestFirmware), ami.UEFIData)
-	assert.Equal(t, []string{instanceKey}, f.keys(t, uefiTestInstance), "the instance's store is left as it was")
-
-	store, err := uefistore.ImageVarStoreFromAMI(ami)
-	require.NoError(t, err)
-	assert.Equal(t, &uefistore.ImageVarStore{ImageID: imageID, Key: ami.UEFIData.Key, Firmware: uefiTestFirmware}, store,
-		"the document decodes into the store a launch seeds from")
-}
-
-// A UEFI instance with no store record or no pointer, and a BIOS instance,
-// give an image that launches from the template.
-func TestCreateImage_AnInstanceWithoutAPointerContributesNothing(t *testing.T) {
-	t.Run("no record", func(t *testing.T) {
-		f := newUEFIImageFixture(t)
-		imageID, err := f.createImage(t, true)
-		require.NoError(t, err)
-		assert.Nil(t, f.ami(t, imageID).UEFIData)
-		assert.Empty(t, f.keys(t, imageID))
-	})
-	t.Run("no pointer", func(t *testing.T) {
-		f := newUEFIImageFixture(t)
-		_, err := f.records.Create(t.Context(), &uefistore.UEFIStoreRecord{
-			InstanceID: uefiTestInstance, AccountID: testAccountID, State: uefistore.StateLive, Generation: 1,
-		})
-		require.NoError(t, err)
-		imageID, err := f.createImage(t, true)
-		require.NoError(t, err)
-		assert.Nil(t, f.ami(t, imageID).UEFIData)
-	})
-	t.Run("BIOS instance", func(t *testing.T) {
-		f := newUEFIImageFixture(t)
-		f.putInstanceStore(t, uefiRegion("never read"))
-		f.svc.SetUEFIStores(nil, "")
-		imageID, err := f.createImage(t, false)
-		require.NoError(t, err)
-		assert.Nil(t, f.ami(t, imageID).UEFIData)
-	})
-}
-
-// A UEFI instance's image is refused rather than made without its store when
-// this node cannot read store records, and a terminating store is refused.
-func TestCreateImage_RefusesWhenTheStoreCannotBeCarried(t *testing.T) {
-	t.Run("no records wired", func(t *testing.T) {
-		f := newUEFIImageFixture(t)
-		f.svc.SetUEFIStores(nil, "")
-		_, err := f.createImage(t, true)
-		require.EqualError(t, err, awserrors.ErrorServerInternal)
-		f.assertNoSnapshot(t)
-	})
-	t.Run("deleting", func(t *testing.T) {
-		f := newUEFIImageFixture(t)
-		f.putInstanceStore(t, uefiRegion("terminating"))
-		_, err := f.records.MarkDeleting(t.Context(), uefiTestInstance)
-		require.NoError(t, err)
-		_, err = f.createImage(t, true)
-		require.EqualError(t, err, awserrors.ErrorIncorrectInstanceState)
-		f.assertNoSnapshot(t)
-	})
-	t.Run("record unreadable", func(t *testing.T) {
-		f := newUEFIImageFixture(t)
-		f.svc.SetUEFIStores(failingRecords{}, "node-a")
-		_, err := f.createImage(t, true)
-		require.EqualError(t, err, awserrors.ErrorServerInternal)
-		f.assertNoSnapshot(t)
-	})
-	t.Run("object unreadable", func(t *testing.T) {
-		f := newUEFIImageFixture(t)
-		key := f.putInstanceStore(t, uefiRegion("x"))
-		_, err := f.objects.PutObject(t.Context(), &awss3.PutObjectInput{
-			Bucket: aws.String(testBucket), Key: aws.String(key), Body: bytes.NewReader([]byte("not gzip")),
-		})
-		require.NoError(t, err)
-		_, err = f.createImage(t, true)
-		require.EqualError(t, err, awserrors.ErrorServerInternal)
-		f.assertNoSnapshot(t)
-	})
-}
-
-// failingRecords cannot read any store record.
-type failingRecords struct{}
-
-func (failingRecords) LeaderGet(context.Context, string) (*uefistore.UEFIStoreRecord, uint64, error) {
-	return nil, 0, assert.AnError
-}
-
-// vanishingRecord shows a live pointer whose object is gone, then no record,
-// as a terminate's reaper leaves it.
-type vanishingRecord struct {
-	stale *uefistore.UEFIStoreRecord
-	reads int
-}
-
-func (v *vanishingRecord) LeaderGet(context.Context, string) (*uefistore.UEFIStoreRecord, uint64, error) {
-	v.reads++
-	if v.reads == 1 {
-		return v.stale, 1, nil
-	}
-	return nil, 0, kvstore.ErrNotFound
-}
-
-// A record that vanishes after CreateImage saw its live pointer is a terminate
-// racing the request, so the image is refused rather than made without its store.
-func TestCreateImage_StoreDeletedMidRequestIsIncorrectInstanceState(t *testing.T) {
-	f := newUEFIImageFixture(t)
-	f.putInstanceStore(t, uefiRegion("x"))
-	stale, _, err := f.records.Get(t.Context(), uefiTestInstance)
-	require.NoError(t, err)
-	stale.Pointer.Key = uefistore.KeyPrefix(uefiTestInstance) + "reaped"
-	f.svc.SetUEFIStores(&vanishingRecord{stale: stale}, "node-a")
-
-	_, err = f.createImage(t, true)
-
-	require.EqualError(t, err, awserrors.ErrorIncorrectInstanceState)
-	f.assertNoSnapshot(t)
-	amis, err := f.svc.MetadataStore().ListAMIs(t.Context())
-	require.NoError(t, err)
-	assert.Empty(t, amis)
-}
-
-// laggingPointer answers the first read with a pointer whose object a prune
-// has removed, and every later read with the record as stored.
-type laggingPointer struct {
-	*uefistore.Store
-
-	stale *uefistore.UEFIStoreRecord
-	reads int
-}
-
-func (l *laggingPointer) LeaderGet(ctx context.Context, id string) (*uefistore.UEFIStoreRecord, uint64, error) {
-	l.reads++
-	if l.reads == 1 {
-		return l.stale, 1, nil
-	}
-	return l.Store.LeaderGet(ctx, id)
-}
-
-// A running instance's write-back can move the pointer and prune the old
-// object between CreateImage's read and its get, so the pointer is read again.
-func TestCreateImage_RereadsAPointerWhoseObjectWasPruned(t *testing.T) {
-	f := newUEFIImageFixture(t)
-	f.putInstanceStore(t, uefiRegion("newest"))
-	stale, _, err := f.records.Get(t.Context(), uefiTestInstance)
-	require.NoError(t, err)
-	stale.Pointer.Key = uefistore.KeyPrefix(uefiTestInstance) + "pruned"
-	lagging := &laggingPointer{Store: f.records, stale: stale}
-	f.svc.SetUEFIStores(lagging, "node-a")
-
-	imageID, err := f.createImage(t, true)
-	require.NoError(t, err)
-
-	assert.Equal(t, 2, lagging.reads)
-	assert.Equal(t, uefiRegion("newest"), f.region(t, f.ami(t, imageID).UEFIData.Key))
-}
-
-// CopyImage gives the copy its own object holding the source's region.
-func TestCopyImage_GivesTheCopyItsOwnUEFIStore(t *testing.T) {
-	f := newUEFIImageFixture(t)
-	f.putInstanceStore(t, uefiRegion("source variables"))
-	src, err := f.createImage(t, true)
-	require.NoError(t, err)
-
-	cp := f.copyImage(t, src, "uefi-copy")
-
-	srcRef, cpRef := f.ami(t, src).UEFIData, f.ami(t, cp).UEFIData
-	require.NotNil(t, cpRef)
-	assert.Equal(t, []string{cpRef.Key}, f.keys(t, cp))
-	assert.Equal(t, srcRef.Firmware, cpRef.Firmware)
-	assert.Equal(t, uefiRegion("source variables"), f.region(t, cpRef.Key))
-	assert.Equal(t, []string{srcRef.Key}, f.keys(t, src), "the source keeps its own object")
-}
-
-// A source AMI with no store gives a copy with none.
-func TestCopyImage_SourceWithoutAUEFIStoreCopiesNone(t *testing.T) {
-	f := newUEFIImageFixture(t)
-	seedCopyableAMI(t, f.objects, "ami-src001", "source-ami", testAccountID, "snap-src001", "vol-src001", 8)
-
-	cp := f.copyImage(t, "ami-src001", "copy")
-
-	assert.Nil(t, f.ami(t, cp).UEFIData)
-	assert.Empty(t, f.keys(t, cp))
-}
-
-// A source store that cannot be copied fails the copy and leaves no AMI.
-func TestCopyImage_UnreadableSourceStoreFails(t *testing.T) {
-	f := newUEFIImageFixture(t)
-	seedCopyableAMI(t, f.objects, "ami-src001", "source-ami", testAccountID, "snap-src001", "vol-src001", 8)
-	src := f.ami(t, "ami-src001")
-	src.UEFIData = uefistore.AMIRef(uefistore.KeyPrefix("ami-src001")+"gone", uefiTestFirmware)
-	require.NoError(t, f.svc.MetadataStore().PutAMI(t.Context(), src))
-
-	_, err := f.svc.CopyImage(t.Context(), validCopyImageServiceInput("ami-src001", "copy"), testAccountID)
-
-	require.EqualError(t, err, awserrors.ErrorServerInternal)
-	amis, err := f.svc.MetadataStore().ListAMIs(t.Context())
-	require.NoError(t, err)
-	assert.Len(t, amis, 1, "no copy is registered without its store")
-}
-
-// TestS7_DeregisteringASourceImageLeavesItsCopysStore enforces ADR-0003:S7
-// for DeregisterImage, which account termination also calls: it deletes only
-// the deregistered image's prefix, and the copy's store is its own.
-func TestS7_DeregisteringASourceImageLeavesItsCopysStore(t *testing.T) {
-	f := newUEFIImageFixture(t)
-	instanceKey := f.putInstanceStore(t, uefiRegion("source variables"))
-	src, err := f.createImage(t, true)
-	require.NoError(t, err)
-	cp := f.copyImage(t, src, "uefi-copy")
-
-	require.NoError(t, f.deregister(t, src))
-
-	assert.Empty(t, f.keys(t, src), "the deregistered image's store is gone")
-	assert.Equal(t, uefiRegion("source variables"), f.region(t, f.ami(t, cp).UEFIData.Key),
-		"ADR-0003:S7: deregistering the source must not delete the store its copy boots from")
-	assert.Equal(t, []string{instanceKey}, f.keys(t, uefiTestInstance), "the instance's store is left as it was")
-}
-
-// failingUEFIDelete fails every delete under the UEFI root.
-type failingUEFIDelete struct{ *objectstore.MemoryObjectStore }
-
-func (s failingUEFIDelete) DeleteObject(ctx context.Context, in *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) {
-	if strings.HasPrefix(aws.StringValue(in.Key), uefistore.ObjectRoot) {
-		return nil, assert.AnError
-	}
-	return s.MemoryObjectStore.DeleteObject(ctx, in)
-}
-
-// DeregisterImage deletes the store before the document, so a failed delete
-// keeps the image for a retry to finish.
-func TestDeregisterImage_FailedUEFIStoreDeleteKeepsTheImage(t *testing.T) {
-	f := newUEFIImageFixture(t)
-	f.putInstanceStore(t, uefiRegion("x"))
-	imageID, err := f.createImage(t, true)
-	require.NoError(t, err)
-	f.svc.store = failingUEFIDelete{f.objects}
-
-	require.EqualError(t, f.deregister(t, imageID), awserrors.ErrorServerInternal)
-	assert.NotNil(t, f.ami(t, imageID).UEFIData, "the document stays for the retry")
-
-	f.svc.store = f.objects
-	require.NoError(t, f.deregister(t, imageID))
-	assert.Empty(t, f.keys(t, imageID))
-	_, err = f.svc.MetadataStore().GetAMI(t.Context(), imageID)
-	assert.True(t, objectstore.IsNoSuchKeyError(err), "the retry removes the document: %v", err)
-}
-
-var _ UEFIStoreRecords = (*laggingPointer)(nil)

spinifex/domains/ec2/instance/provider_efi_volume_test.go

@@ -0,0 +1,126 @@
+package instance
+
+import (
+	"bytes"
+	"context"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// varsTemplateBytes stands in for a firmware VARS template. The real one is
+// 540,672 bytes, but nothing here depends on the size beyond it being exact.
+func varsTemplateBytes() []byte { return bytes.Repeat([]byte{0xAB}, 4096) }
+
+func providerEFIService(t *testing.T, provider ebsprovider.EBSProvider) *InstanceServiceImpl {
+	t.Helper()
+	return providerRootVolumeService(t, provider, rootVolumeAMILoader(), objectstore.NewMemoryObjectStore())
+}
+
+// TestCreateEFIVolume_Provider_SeedsFromTemplate is the core of the EFI
+// decouple: the variable store is allocated through the provider with the
+// template as seed bytes, so no control-plane engine is built for it.
+func TestCreateEFIVolume_Provider_SeedsFromTemplate(t *testing.T) {
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
+	svc := providerEFIService(t, provider)
+	template := varsTemplateBytes()
+
+	require.NoError(t, svc.createEFIVolumeViaProvider(context.Background(), "vol-efiseed001-efi", template))
+
+	volume, err := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-efiseed001-efi",
+	})
+	require.NoError(t, err)
+	assert.Equal(t, int64(len(template)), volume.CapacityBytes,
+		"the store must be sized byte-exactly to the VARS template")
+
+	seed, ok := provider.SeedData("vol-efiseed001-efi")
+	require.True(t, ok)
+	assert.Equal(t, template, seed, "the template must reach the provider as seed bytes")
+}
+
+// A provider that rounds capacity to a GiB boundary produces a VARS region
+// pflash refuses, which would surface as an unbootable guest rather than a
+// failed launch. The mismatch has to fail the launch instead.
+func TestCreateEFIVolume_Provider_CapacityMismatchFails(t *testing.T) {
+	provider := &roundingProvider{
+		EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true}),
+	}
+	svc := providerEFIService(t, provider)
+
+	err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efiround01-efi", varsTemplateBytes())
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+}
+
+// A relaunch reissues the same create. Reseeding would overwrite the BootOrder
+// the guest wrote on its first boot, so the second call must be a no-op.
+func TestCreateEFIVolume_Provider_RepeatDoesNotReseed(t *testing.T) {
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
+	svc := providerEFIService(t, provider)
+	ctx := context.Background()
+	template := varsTemplateBytes()
+
+	require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template))
+	require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template))
+
+	seed, ok := provider.SeedData("vol-efirepeat1-efi")
+	require.True(t, ok)
+	assert.Equal(t, template, seed)
+}
+
+// A seed above the wire bound must fail the launch with an error naming the
+// limit, not reach NATS and fail as an oversized publish.
+func TestCreateEFIVolume_Provider_OversizedTemplateFails(t *testing.T) {
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
+	svc := providerEFIService(t, provider)
+
+	err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efitoobig1-efi", make([]byte, ebsprovider.MaxSeedBytes+1))
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+}
+
+// TestPrepareEFIVolume_Provider_RegistersEBSRequest covers the whole function
+// against this host's real firmware, including the name the VM launches with.
+func TestPrepareEFIVolume_Provider_RegistersEBSRequest(t *testing.T) {
+	if _, _, _, err := vm.FirmwarePaths("x86_64"); err != nil {
+		t.Skipf("no UEFI firmware on this host: %v", err)
+	}
+
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
+	svc := providerEFIService(t, provider)
+	instance := &vm.VM{}
+
+	require.NoError(t, svc.prepareEFIVolume(context.Background(), "vol-efiprepare001", instance, "x86_64"))
+
+	instance.EBSRequests.Mu.Lock()
+	defer instance.EBSRequests.Mu.Unlock()
+	require.Len(t, instance.EBSRequests.Requests, 1)
+	assert.Equal(t, "vol-efiprepare001-efi", instance.EBSRequests.Requests[0].Name)
+	assert.True(t, instance.EBSRequests.Requests[0].EFI)
+	assert.False(t, instance.EBSRequests.Requests[0].Boot)
+
+	seed, ok := provider.SeedData("vol-efiprepare001-efi")
+	require.True(t, ok)
+	assert.NotEmpty(t, seed, "the store must be seeded from this host's VARS template")
+}
+
+// roundingProvider reports a GiB-rounded capacity the way a provider with no
+// sub-GiB volume granularity would.
+type roundingProvider struct {
+	ebsprovider.EBSProvider
+}
+
+func (p *roundingProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) {
+	volume, err := p.EBSProvider.CreateVolume(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	volume.CapacityBytes = 1 << 30
+	return volume, nil
+}

spinifex/domains/ec2/instance/service_impl.go

@@ -10,6 +10,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
 	"log/slog"
 	"maps"
+	"os"
 	"slices"
 	"sort"
 	"strconv"
@@ -26,7 +27,6 @@ import (
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	ebspolicy "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/policy"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instancetypes"
 	ec2platform "github.com/mulgadc/spinifex/spinifex/domains/ec2/platform"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
@@ -1265,10 +1265,6 @@ func (s *InstanceServiceImpl) StartInstance(ctx context.Context, instance *vm.VM
 				slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
 				return refusal
 			}
-			if errors.Is(err, vm.ErrUEFIClaimHeld) {
-				slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
-				return errInsufficientCapacity
-			}
 			slog.ErrorContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
 			return errors.New(awserrors.ErrorServerInternal)
 		}
@@ -1616,20 +1612,18 @@ func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.Ru
 		return nil, err
 	}
 
-	// Step 2: Seed the UEFI variable store, which is not a volume. A failure
-	// terminates the instance, and terminate deletes whatever store it left.
-	image, err := s.imageUEFIVarStore(ctx, instance, aws.StringValue(input.ImageId))
-	if err != nil {
-		slog.ErrorContext(ctx, "Failed to read the image's UEFI variable store", "instanceId", instance.ID,
-			"imageId", aws.StringValue(input.ImageId), "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-	if err := s.vmMgr.SeedUEFIVarStore(ctx, instance, image); err != nil {
-		slog.ErrorContext(ctx, "Failed to seed the UEFI variable store", "instanceId", instance.ID, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
+	// Step 2: Create EFI variable store (only when the AMI is UEFI; BIOS
+	// guests must not allocate an orphan VARS volume).
+	if instance.BootMode == "uefi" || instance.BootMode == "uefi-preferred" {
+		arch := instanceArchitecture(s.instanceTypes[*input.InstanceType])
+		err = s.prepareEFIVolume(ctx, imageId, instance, arch)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	// Every volume the request named is reported.
+	// The EFI store is internal, so it is not reported; every volume the
+	// request named is.
 	volumeInfos := []VolumeInfo{
 		{
 			VolumeId:            imageId,
@@ -1649,23 +1643,6 @@ func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.Ru
 	return volumeInfos, nil
 }
 
-// imageUEFIVarStore returns the store the launching AMI names for a UEFI
-// instance to seed from, or nil when the instance boots without UEFI or the
-// AMI names none.
-func (s *InstanceServiceImpl) imageUEFIVarStore(ctx context.Context, instance *vm.VM, imageID string) (*uefistore.ImageVarStore, error) {
-	if !vm.UsesUEFIVarStore(instance) || !strings.HasPrefix(imageID, "ami-") {
-		return nil, nil
-	}
-	if s.amiLoader == nil {
-		return nil, errors.New("AMI loader not initialized")
-	}
-	ami, err := s.amiLoader.GetAMIConfig(ctx, imageID)
-	if err != nil {
-		return nil, err
-	}
-	return uefistore.ImageVarStoreFromAMI(ami)
-}
-
 // prepareDataVolumes creates an empty volume for every EBS mapping that is not
 // the root and records each as a non-boot request the launcher attaches. A
 // failure unwinds the volumes this call created, so a refused launch leaves none.
@@ -1908,6 +1885,81 @@ func rootVolumeIOPS(requested int) int {
 	return requested
 }
 
+// prepareEFIVolume creates the per-VM EFI variable store, sized exactly to the
+// firmware VARS template (pflash requires byte-exact size) and seeded from it.
+// arch is "x86_64" | "arm64".
+func (s *InstanceServiceImpl) prepareEFIVolume(ctx context.Context, volumeID string, instance *vm.VM, arch string) error {
+	codePath, varsTemplate, varsSize, err := vm.FirmwarePaths(arch)
+	if err != nil {
+		slog.ErrorContext(ctx, "UEFI firmware not installed on this host", "arch", arch, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	template, err := os.ReadFile(varsTemplate)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to read VARS template", "path", varsTemplate, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	if int64(len(template)) != varsSize {
+		slog.ErrorContext(ctx, "VARS template size mismatch between stat and read", "path", varsTemplate, "statSize", varsSize, "readSize", len(template))
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	slog.InfoContext(ctx, "Preparing EFI variable store", "arch", arch, "firmwarePath", codePath, "varsTemplate", varsTemplate, "size", varsSize)
+
+	efiVolumeName := fmt.Sprintf("%s-efi", volumeID)
+	if s.ebsProvider == nil {
+		slog.ErrorContext(ctx, "no EBS provider configured", "op", "prepareEFIVolume")
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	if err := s.createEFIVolumeViaProvider(ctx, efiVolumeName, template); err != nil {
+		return err
+	}
+
+	instance.EBSRequests.Mu.Lock()
+	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
+		Name: efiVolumeName,
+		Boot: false,
+		EFI:  true,
+	})
+	instance.EBSRequests.Mu.Unlock()
+
+	return nil
+}
+
+// createEFIVolumeViaProvider allocates the EFI variable store through the
+// provider, seeding it with this node's firmware VARS template. The bytes
+// travel with the request so the region matches the firmware that will run.
+func (s *InstanceServiceImpl) createEFIVolumeViaProvider(ctx context.Context, efiVolumeName string, template []byte) error {
+	slog.InfoContext(ctx, "Creating EFI variable store via EBS provider", "name", efiVolumeName, "seedBytes", len(template))
+
+	// The store is internal to the launch, never attachable and filtered out of
+	// DescribeVolumes, so it gets no ebsmetadata document the way a root volume
+	// does. Its lifecycle is the root volume's DeleteVolume.
+	created, err := s.ebsProvider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
+		Versioned:        ebsprovider.NewVersioned(),
+		VolumeID:         efiVolumeName,
+		CapacityRange:    ebsprovider.CapacityRange{RequiredBytes: int64(len(template))},
+		AvailabilityZone: s.config.AZ,
+		SeedData:         template,
+	})
+	if err != nil {
+		slog.ErrorContext(ctx, "Provider EFI volume creation failed", "name", efiVolumeName, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	if created == nil {
+		slog.ErrorContext(ctx, "Provider returned no volume for the EFI variable store", "name", efiVolumeName)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+
+	// pflash rejects a VARS region that is not byte-exact, so a provider that
+	// rounded the capacity up to a GiB boundary yields a guest that cannot boot.
+	if created.CapacityBytes != int64(len(template)) {
+		slog.ErrorContext(ctx, "Provider EFI volume capacity does not match the VARS template",
+			"name", efiVolumeName, "wantBytes", len(template), "gotBytes", created.CapacityBytes)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	return nil
+}
+
 // checkImageArchitecture refuses an AMI whose architecture the instance type
 // does not support, as AWS does. An unrecorded architecture on either side is
 // not a mismatch.
@@ -2708,7 +2760,7 @@ func (s *InstanceServiceImpl) capacityRefusal(ctx context.Context, instance *vm.
 func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instance *vm.VM, undo func()) error {
 	// Before anything is stamped or claimed, so the record handed back is the
 	// one that was claimed.
-	if err := s.vmMgr.CheckUEFIVarStore(ctx, instance); err != nil {
+	if err := vm.CheckUEFIVarStore(instance); err != nil {
 		slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without exactly one variable store",
 			"instanceId", instance.ID, "err", err)
 		undo()
@@ -2841,10 +2893,6 @@ func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instanc
 		if refusal := volumeHeldElsewhereError(err); refusal != nil {
 			return refusal
 		}
-		// A live node holds the variable store, so another node may take it.
-		if errors.Is(err, vm.ErrUEFIClaimHeld) {
-			return s.capacityRefusal(ctx, instance)
-		}
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 

spinifex/domains/ec2/instance/uefi_seed_test.go

@@ -1,171 +0,0 @@
-package instance
-
-import (
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const testSeedInstance = "i-0a1b2c3d4e5f60718"
-
-type seedInstanceTypes map[string]vm.InstanceTypeSpec
-
-func (r seedInstanceTypes) Resolve(name string) (vm.InstanceTypeSpec, bool) {
-	spec, ok := r[name]
-	return spec, ok
-}
-
-// uefiSeedingService is an instance service whose VM manager seeds stores into
-// a real JetStream's store records and claims and an in-memory object store,
-// under a stand-in x86_64 firmware.
-func uefiSeedingService(t *testing.T) (*InstanceServiceImpl, *ebsprovider.MemoryProvider, *uefistore.Store, *objectstore.MemoryObjectStore) {
-	t.Helper()
-	fwDir := t.TempDir()
-	code := filepath.Join(fwDir, "OVMF_CODE.fd")
-	vars := filepath.Join(fwDir, "OVMF_VARS.fd")
-	require.NoError(t, os.WriteFile(code, []byte("code"), 0o600))
-	require.NoError(t, os.WriteFile(vars, make([]byte, 4096), 0o600))
-	orig, had := vm.FirmwarePathCandidates["x86_64"]
-	vm.FirmwarePathCandidates["x86_64"] = []vm.FirmwareCandidate{{Code: code, VarsTemplate: vars}}
-	t.Cleanup(func() {
-		if had {
-			vm.FirmwarePathCandidates["x86_64"] = orig
-		} else {
-			delete(vm.FirmwarePathCandidates, "x86_64")
-		}
-	})
-
-	_, _, js := testutil.StartTestJetStream(t)
-	records := uefistore.NewStore(js)
-	require.NoError(t, records.Open(t.Context()))
-
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{CrashConsistentSnapshot: true, VolumeEnumeration: true})
-	seedProviderSnapshot(t, provider, "vol-origin", "snap-source")
-	objects := objectstore.NewMemoryObjectStore()
-	svc := providerRootVolumeService(t, provider, rootVolumeAMILoader(), objects)
-	svc.vmMgr = vm.NewManagerWithDeps(vm.Deps{
-		NodeID:        "node-a",
-		InstanceTypes: seedInstanceTypes{"t3.nano": {Architecture: "x86_64", VCPUs: 1, MemoryMiB: 512}},
-		UEFIVarStore: vm.UEFIVarStoreDeps{
-			Objects: objects, Bucket: testRootBucket, DataDir: t.TempDir(), Records: records,
-			Claims: uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a"),
-		},
-	})
-	return svc, provider, records, objects
-}
-
-func uefiLaunch() (*ec2.RunInstancesInput, *vm.VM) {
-	input := &ec2.RunInstancesInput{ImageId: aws.String("ami-1"), InstanceType: aws.String("t3.nano")}
-	return input, &vm.VM{ID: testSeedInstance, AccountID: testRootAccount, InstanceType: "t3.nano", BootMode: "uefi"}
-}
-
-// A UEFI launch seeds its store as an object and a store record, and creates
-// no <root>-efi volume and no EFI request.
-func TestGenerateVolumes_UEFILaunchSeedsAStoreAndCreatesNoEFIVolume(t *testing.T) {
-	svc, provider, records, objects := uefiSeedingService(t)
-	input, instance := uefiLaunch()
-
-	infos, err := svc.GenerateVolumes(t.Context(), input, instance)
-	require.NoError(t, err)
-
-	require.Len(t, infos, 1, "only the root volume is reported")
-	instance.EBSRequests.Mu.Lock()
-	for _, req := range instance.EBSRequests.Requests {
-		assert.False(t, req.EFI, "no EFI request is appended")
-		assert.False(t, strings.HasSuffix(req.Name, "-efi"), req.Name)
-	}
-	instance.EBSRequests.Mu.Unlock()
-	listed, err := provider.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
-	require.NoError(t, err)
-	for _, v := range listed.Volumes {
-		assert.False(t, strings.HasSuffix(v.ID, "-efi"), "no EFI volume is created: %s", v.ID)
-	}
-
-	rec, _, err := records.Get(t.Context(), testSeedInstance)
-	require.NoError(t, err)
-	assert.Equal(t, uefistore.StateLive, rec.State)
-	require.NotNil(t, rec.Pointer)
-	keys, err := uefistore.ListObjects(t.Context(), objects, testRootBucket, testSeedInstance)
-	require.NoError(t, err)
-	assert.Equal(t, []string{rec.Pointer.Key}, keys, "the pointer names the one template object")
-}
-
-// A seed that cannot be written fails the launch as InternalError, and no EFI
-// volume is created in its place.
-func TestGenerateVolumes_FailedSeedIsInternalError(t *testing.T) {
-	svc, provider, _, _ := uefiSeedingService(t)
-	svc.vmMgr = vm.NewManager()
-	input, instance := uefiLaunch()
-
-	_, err := svc.GenerateVolumes(t.Context(), input, instance)
-
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-	listed, err := provider.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
-	require.NoError(t, err)
-	for _, v := range listed.Volumes {
-		assert.False(t, strings.HasSuffix(v.ID, "-efi"), v.ID)
-	}
-}
-
-// putImageStore gives ami-1 a store object holding vars and names it in the
-// AMI document the launch reads.
-func putImageStore(t *testing.T, svc *InstanceServiceImpl, objects *objectstore.MemoryObjectStore, key string, vars []byte) {
-	t.Helper()
-	if strings.HasPrefix(key, uefistore.KeyPrefix("ami-1")) {
-		require.NoError(t, uefistore.PutObject(t.Context(), objects, testRootBucket, key, vars, uefistore.ObjectMeta{
-			OwnerID: "ami-1", WriterNode: "node-b", Source: uefistore.SourceInstance("i-0000000000000000a"),
-			Firmware: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: int64(len(vars)), CodeSHA256: "other"},
-		}))
-	}
-	loader := svc.amiLoader.(*fakeAMILoader)
-	ami := loader.byID["ami-1"]
-	ami.UEFIData = uefistore.AMIRef(key, uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: int64(len(vars)), CodeSHA256: "other"})
-	loader.byID["ami-1"] = ami
-}
-
-// An AMI that names a store seeds the new instance's own object from it.
-func TestGenerateVolumes_UEFILaunchSeedsFromTheAMIsStore(t *testing.T) {
-	svc, _, records, objects := uefiSeedingService(t)
-	vars := make([]byte, 4096)
-	copy(vars, "image variables")
-	putImageStore(t, svc, objects, uefistore.KeyPrefix("ami-1")+"k", vars)
-	input, instance := uefiLaunch()
-
-	_, err := svc.GenerateVolumes(t.Context(), input, instance)
-	require.NoError(t, err)
-
-	rec, _, err := records.Get(t.Context(), testSeedInstance)
-	require.NoError(t, err)
-	require.NotNil(t, rec.Pointer)
-	got, _, err := uefistore.ReadObject(t.Context(), objects, testRootBucket, rec.Pointer.Key, 4096)
-	require.NoError(t, err)
-	assert.Equal(t, vars, got)
-}
-
-// An AMI document whose store reference is outside the image's own prefix is
-// refused, and the launch records no store.
-func TestGenerateVolumes_AMIStoreOutsideItsPrefixIsInternalError(t *testing.T) {
-	svc, _, records, objects := uefiSeedingService(t)
-	putImageStore(t, svc, objects, uefistore.KeyPrefix("i-0000000000000000a")+"k", make([]byte, 4096))
-	input, instance := uefiLaunch()
-
-	_, err := svc.GenerateVolumes(t.Context(), input, instance)
-
-	require.EqualError(t, err, awserrors.ErrorServerInternal)
-	_, _, err = records.Get(t.Context(), testSeedInstance)
-	require.ErrorIs(t, err, kvstore.ErrNotFound)
-}

spinifex/domains/ec2/instance/uefistore/cleanup.go

@@ -10,6 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -25,9 +26,9 @@ func (s *Store) Exists(ctx context.Context, instanceID string) (bool, error) {
 	return s.kv.Exists(ctx, instanceID)
 }
 
-// MarkDeleting records that instanceID's store is to be deleted. It reports
-// false with no error when the instance has no store record. Only a terminate
-// or an operator's forget may call it.
+// MarkDeleting records that instanceID's store is to be deleted, and drops any
+// write-back still owed. It reports false with no error when the instance has
+// no store record. Only a terminate or an operator's forget may call it.
 func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, error) {
 	for range markAttempts {
 		rec, rev, err := s.Get(ctx, instanceID)
@@ -41,6 +42,7 @@ func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, erro
 			return true, nil
 		}
 		rec.State = StateDeleting
+		rec.WriteBack = nil
 		_, err = s.CompareAndSet(ctx, rec, rev)
 		if err == nil {
 			return true, nil
@@ -70,7 +72,7 @@ func (s *Store) InstanceIDs(ctx context.Context) ([]string, error) {
 
 	var ids []string
 	for key := range lister.Keys() {
-		if key != kvstore.VersionKey {
+		if key != kvutil.VersionKey {
 			ids = append(ids, key)
 		}
 	}

spinifex/domains/ec2/instance/uefistore/cleanup_test.go

@@ -37,7 +37,7 @@ func keysUnder(t *testing.T, objects objectstore.ObjectStore, prefix string) []s
 }
 
 // Deletion is final: a writer holding the revision of a deleting record, such
-// as a late pointer move, must not bring the store back to live.
+// as a write-back retry, must not bring the store back to live.
 func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	s, _ := openStore(t)
 	rev, err := s.Create(t.Context(), liveRecord())
@@ -47,7 +47,8 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	rev, err = s.CompareAndSet(t.Context(), deleting, rev)
 	require.NoError(t, err)
 
-	revived := pointedRecord()
+	revived := liveRecord()
+	revived.WriteBack = writeBack()
 	_, err = s.CompareAndSet(t.Context(), revived, rev)
 	require.ErrorIs(t, err, uefistore.ErrDeleting)
 
@@ -57,9 +58,24 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	assert.Equal(t, uefistore.StateDeleting, got.State)
 }
 
-func TestMarkDeleting_MovesLiveToDeletingAndKeepsWhatItNames(t *testing.T) {
+// A write-back owed on a deleting store would put an object behind the delete.
+func TestCompareAndSet_RefusesDeletingWithAWriteBack(t *testing.T) {
 	s, _ := openStore(t)
-	_, err := s.Create(t.Context(), pointedRecord())
+	rev, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	rec := liveRecord()
+	rec.State = uefistore.StateDeleting
+	rec.WriteBack = writeBack()
+
+	_, err = s.CompareAndSet(t.Context(), rec, rev)
+	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+}
+
+func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+	_, err := s.Create(t.Context(), rec)
 	require.NoError(t, err)
 
 	has, err := s.MarkDeleting(t.Context(), instanceID)
@@ -69,8 +85,8 @@ func TestMarkDeleting_MovesLiveToDeletingAndKeepsWhatItNames(t *testing.T) {
 	got, rev, err := s.Get(t.Context(), instanceID)
 	require.NoError(t, err)
 	assert.Equal(t, uefistore.StateDeleting, got.State)
+	assert.Nil(t, got.WriteBack)
 	assert.Equal(t, "111122223333", got.AccountID)
-	assert.Equal(t, pointer("b"), got.Pointer, "the pointer stays readable until the prefix is gone")
 
 	has, err = s.MarkDeleting(t.Context(), instanceID)
 	require.NoError(t, err)

spinifex/domains/ec2/instance/uefistore/fence.go

@@ -1,93 +0,0 @@
-package uefistore
-
-import (
-	"context"
-	"errors"
-	"fmt"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
-)
-
-// ClaimPrefix is the kvsync prefix of every store's claim. An instance's claim
-// is ClaimPrefix+instanceID, and names the node that may hold variable changes
-// no object has yet.
-const ClaimPrefix = "uefi."
-
-// NewClaims returns node's group of store claims over store. The daemon builds
-// it once per process, as kvsync requires of every prefix.
-func NewClaims(store kvsync.KVStore, node string) *kvsync.ClaimGroup {
-	return kvsync.NewClaimGroup(store, ClaimPrefix, node)
-}
-
-// ErrFenced is a stamp or pointer move refused because the record carries a
-// claim generation other than the caller's: a later acquisition has stamped
-// it, so the caller no longer owns the store and must change nothing.
-var ErrFenced = errors.New("uefistore: store record is fenced to another claim generation")
-
-// fenceAttempts bounds the read-then-write loops of Stamp and MovePointer. A
-// lost race is re-read, so a concurrent stamp is seen and refused, not retried.
-const fenceAttempts = 5
-
-// Stamp raises instanceID's record to generation, the claim token the caller
-// has just acquired or adopted, and returns the record it stamped, whose
-// pointer the caller boots from. One above generation is ErrFenced, and one
-// deleting is ErrDeleting. A record already at generation is still written by
-// CAS, because a follower can answer the read with an older pointer.
-func (s *Store) Stamp(ctx context.Context, instanceID string, generation uint64) (*UEFIStoreRecord, error) {
-	if generation == 0 {
-		return nil, fmt.Errorf("uefistore: stamp %s: generation 0 is no claim", instanceID)
-	}
-	for range fenceAttempts {
-		rec, rev, err := s.Get(ctx, instanceID)
-		if err != nil {
-			return nil, err
-		}
-		switch {
-		case rec.State == StateDeleting:
-			return nil, fmt.Errorf("%w: %s", ErrDeleting, instanceID)
-		case rec.Generation > generation:
-			return nil, fmt.Errorf("%w: %s is at generation %d, above the claim's %d", ErrFenced, instanceID, rec.Generation, generation)
-		}
-		rec.Generation = generation
-		_, err = s.CompareAndSet(ctx, rec, rev)
-		if err == nil {
-			return rec, nil
-		}
-		if !errors.Is(err, kvstore.ErrConflict) {
-			return nil, err
-		}
-	}
-	return nil, fmt.Errorf("uefistore: stamp %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
-}
-
-// MovePointer points instanceID's record at ref only while the record is at
-// generation. A record at any other generation is ErrFenced and is left
-// unchanged. The leader answers the read, because a caller drops its copy on
-// ErrFenced or ErrDeleting, and the CAS keeps a lost race from succeeding.
-func (s *Store) MovePointer(ctx context.Context, instanceID string, generation uint64, ref UEFIVarStoreRef) error {
-	if err := ref.validate(instanceID); err != nil {
-		return fmt.Errorf("%w: move %s: %w", ErrInvalidRecord, instanceID, err)
-	}
-	for range fenceAttempts {
-		rec, rev, err := s.LeaderGet(ctx, instanceID)
-		if err != nil {
-			return err
-		}
-		switch {
-		case rec.State == StateDeleting:
-			return fmt.Errorf("%w: %s", ErrDeleting, instanceID)
-		case rec.Generation != generation:
-			return fmt.Errorf("%w: %s is at generation %d, and the writer holds %d", ErrFenced, instanceID, rec.Generation, generation)
-		}
-		rec.Pointer = &ref
-		_, err = s.CompareAndSet(ctx, rec, rev)
-		if err == nil {
-			return nil
-		}
-		if !errors.Is(err, kvstore.ErrConflict) {
-			return err
-		}
-	}
-	return fmt.Errorf("uefistore: move %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
-}

spinifex/domains/ec2/instance/uefistore/fence_test.go

@@ -1,213 +0,0 @@
-package uefistore_test
-
-import (
-	"testing"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func revisionOf(t *testing.T, s *uefistore.Store) uint64 {
-	t.Helper()
-	_, rev, err := s.Get(t.Context(), instanceID)
-	require.NoError(t, err)
-	return rev
-}
-
-func TestStamp_RaisesTheGenerationAndReturnsThePointerToBootFrom(t *testing.T) {
-	s, _ := openStore(t)
-	_, err := s.Create(t.Context(), pointedRecord())
-	require.NoError(t, err)
-
-	rec, err := s.Stamp(t.Context(), instanceID, 9)
-	require.NoError(t, err)
-	assert.Equal(t, uint64(9), rec.Generation)
-	assert.Equal(t, pointer("b"), rec.Pointer)
-
-	got, _, err := s.Get(t.Context(), instanceID)
-	require.NoError(t, err)
-	assert.Equal(t, rec, got, "the returned record is the one stored")
-}
-
-// A daemon restart adopts its claim at the same generation. The stamp still
-// writes by CAS, so a pointer read from a lagging follower fails the write and
-// is read again rather than booted from.
-func TestStamp_AtTheRecordsGenerationConfirmsTheReadByCAS(t *testing.T) {
-	s, _ := openStore(t)
-	_, err := s.Create(t.Context(), pointedRecord())
-	require.NoError(t, err)
-	before := revisionOf(t, s)
-
-	rec, err := s.Stamp(t.Context(), instanceID, 7)
-	require.NoError(t, err)
-	assert.Equal(t, pointer("b"), rec.Pointer)
-	assert.Equal(t, uint64(7), rec.Generation)
-	assert.Greater(t, revisionOf(t, s), before, "the read must be confirmed by a revision-checked write")
-}
-
-func TestStamp_RefusesAndChangesNothing(t *testing.T) {
-	tests := []struct {
-		name       string
-		deleting   bool
-		generation uint64
-		want       error
-	}{
-		{name: "a later acquisition stamped it", generation: 6, want: uefistore.ErrFenced},
-		{name: "the store is being deleted", deleting: true, generation: 8, want: uefistore.ErrDeleting},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s, _ := openStore(t)
-			rev, err := s.Create(t.Context(), pointedRecord())
-			require.NoError(t, err)
-			if tt.deleting {
-				_, err = s.MarkDeleting(t.Context(), instanceID)
-				require.NoError(t, err)
-				rev = revisionOf(t, s)
-			}
-
-			_, err = s.Stamp(t.Context(), instanceID, tt.generation)
-			require.ErrorIs(t, err, tt.want)
-			assert.Equal(t, rev, revisionOf(t, s))
-		})
-	}
-}
-
-func TestStamp_MissingRecordIsNotFound(t *testing.T) {
-	s, _ := openStore(t)
-	_, err := s.Stamp(t.Context(), instanceID, 1)
-	require.ErrorIs(t, err, kvstore.ErrNotFound)
-}
-
-// The record keeps only the current version, so each move replaces the
-// pointer and names nothing it replaced.
-func TestMovePointer_ReplacesThePointer(t *testing.T) {
-	s, _ := openStore(t)
-	_, err := s.Create(t.Context(), pointedRecord())
-	require.NoError(t, err)
-
-	for _, name := range []string{"c", "d"} {
-		require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer(name)))
-	}
-
-	got, _, err := s.Get(t.Context(), instanceID)
-	require.NoError(t, err)
-	assert.Equal(t, pointer("d"), got.Pointer)
-	assert.Equal(t, uint64(7), got.Generation)
-}
-
-// A move to the key the record already names is still a CAS at the revision
-// read, so a fenced writer that read a lagging follower cannot see success.
-func TestMovePointer_ToTheCurrentKeyConfirmsTheReadByCAS(t *testing.T) {
-	s, _ := openStore(t)
-	_, err := s.Create(t.Context(), pointedRecord())
-	require.NoError(t, err)
-	before := revisionOf(t, s)
-
-	require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer("b")))
-	assert.Greater(t, revisionOf(t, s), before)
-}
-
-func TestMovePointer_RefusesAndChangesNothing(t *testing.T) {
-	tests := []struct {
-		name       string
-		deleting   bool
-		generation uint64
-		ref        *uefistore.UEFIVarStoreRef
-		want       error
-	}{
-		{name: "an earlier generation", generation: 6, ref: pointer("c"), want: uefistore.ErrFenced},
-		{name: "a later generation than the stamp", generation: 8, ref: pointer("c"), want: uefistore.ErrFenced},
-		{name: "a deleting store", deleting: true, generation: 7, ref: pointer("c"), want: uefistore.ErrDeleting},
-		{name: "a key under another instance", generation: 7,
-			ref:  &uefistore.UEFIVarStoreRef{Key: uefistore.KeyPrefix("i-0fedcba9876543210") + "c", Firmware: pointer("c").Firmware, WriterNode: "n"},
-			want: uefistore.ErrInvalidRecord},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s, _ := openStore(t)
-			_, err := s.Create(t.Context(), pointedRecord())
-			require.NoError(t, err)
-			if tt.deleting {
-				_, err = s.MarkDeleting(t.Context(), instanceID)
-				require.NoError(t, err)
-			}
-			rev := revisionOf(t, s)
-
-			err = s.MovePointer(t.Context(), instanceID, tt.generation, *tt.ref)
-			require.ErrorIs(t, err, tt.want)
-			got, gotRev, err := s.Get(t.Context(), instanceID)
-			require.NoError(t, err)
-			assert.Equal(t, rev, gotRev)
-			assert.Equal(t, pointer("b"), got.Pointer)
-		})
-	}
-}
-
-// claimedStore is a store record and a claim group per node over one
-// JetStream, as two daemons see them.
-type claimedStore struct {
-	records *uefistore.Store
-	a, b    *kvsync.Claim
-}
-
-func newClaimedStore(t *testing.T) *claimedStore {
-	t.Helper()
-	_, _, js := testutil.StartTestJetStream(t)
-	records := uefistore.NewStore(js)
-	require.NoError(t, records.Open(t.Context()))
-	_, err := records.Create(t.Context(), liveRecord())
-	require.NoError(t, err)
-	return &claimedStore{
-		records: records,
-		a:       uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a").Get(instanceID),
-		b:       uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-b").Get(instanceID),
-	}
-}
-
-// A node partitioned long enough to be taken over is fenced by the new
-// owner's stamp: its pointer move fails and the record keeps what it had.
-func TestFence_TheOldHoldersMoveFailsAfterTheNewOwnersStamp(t *testing.T) {
-	c := newClaimedStore(t)
-	first, err := c.a.TryLock(t.Context())
-	require.NoError(t, err)
-	_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
-	require.NoError(t, err)
-	require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("b")))
-
-	second, err := c.b.TakeOver(t.Context(), first)
-	require.NoError(t, err)
-	_, err = c.records.Stamp(t.Context(), instanceID, second.Generation)
-	require.NoError(t, err)
-	rev := revisionOf(t, c.records)
-
-	err = c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("late"))
-	require.ErrorIs(t, err, uefistore.ErrFenced)
-	got, gotRev, err := c.records.Get(t.Context(), instanceID)
-	require.NoError(t, err)
-	assert.Equal(t, rev, gotRev)
-	assert.Equal(t, pointer("b"), got.Pointer)
-	assert.Equal(t, second.Generation, got.Generation)
-}
-
-// The new owner reads the pointer after it stamps, so a move the old holder
-// landed between the takeover and the stamp is the one it boots from.
-func TestFence_AMoveBetweenTakeoverAndStampIsThePointerTheNewOwnerBoots(t *testing.T) {
-	c := newClaimedStore(t)
-	first, err := c.a.TryLock(t.Context())
-	require.NoError(t, err)
-	_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
-	require.NoError(t, err)
-
-	second, err := c.b.TakeOver(t.Context(), first)
-	require.NoError(t, err)
-	require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("landed")))
-
-	rec, err := c.records.Stamp(t.Context(), instanceID, second.Generation)
-	require.NoError(t, err)
-	assert.Equal(t, keyOf("landed"), rec.Pointer.Key)
-}

spinifex/domains/ec2/instance/uefistore/image.go

@@ -1,145 +0,0 @@
-package uefistore
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"strings"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-)
-
-const (
-	// GetTimeout bounds one get of a store object, a fixed-size region.
-	GetTimeout = 10 * time.Second
-
-	// bodySlack is how far a compressed body may exceed its region before it
-	// is refused; gzip grows incompressible data by far less.
-	bodySlack = 1 << 20
-)
-
-// ErrObjectNotFound is a store object that does not exist.
-var ErrObjectNotFound = errors.New("uefistore: store object does not exist")
-
-// SourceImage is the source of a store copied from imageID's store.
-func SourceImage(imageID string) string { return "image:" + imageID }
-
-// SourceInstance is the source of an image store copied from instanceID's.
-func SourceInstance(instanceID string) string { return "instance:" + instanceID }
-
-// ImageVarStore is the store an AMI document names: one object under the
-// image's own prefix and the firmware it was written under.
-type ImageVarStore struct {
-	ImageID  string
-	Key      string
-	Firmware FirmwareIdentity
-}
-
-// ImageVarStoreFromAMI returns the store ami names, or nil when it names none.
-// A reference whose key is not one object under the image's own prefix, or
-// that names no firmware, is ErrInvalidObject.
-func ImageVarStoreFromAMI(ami ebsmetadata.AMI) (*ImageVarStore, error) {
-	ref := ami.UEFIData
-	if ref == nil {
-		return nil, nil
-	}
-	if !strings.HasPrefix(ami.ImageID, "ami-") || !ownsKey(ami.ImageID, ref.Key) {
-		return nil, fmt.Errorf("%w: %s names UEFI store %q outside %s", ErrInvalidObject, ami.ImageID, ref.Key, KeyPrefix(ami.ImageID))
-	}
-	if ref.Firmware.Arch == "" || ref.Firmware.VarsSize <= 0 {
-		return nil, fmt.Errorf("%w: %s names UEFI store %q with no firmware arch and VARS size", ErrInvalidObject, ami.ImageID, ref.Key)
-	}
-	return &ImageVarStore{
-		ImageID: ami.ImageID,
-		Key:     ref.Key,
-		Firmware: FirmwareIdentity{
-			Arch: ref.Firmware.Arch, VarsSize: ref.Firmware.VarsSize, CodeSHA256: ref.Firmware.CodeSHA256,
-		},
-	}, nil
-}
-
-// AMIRef returns the reference an AMI document records for key under fw.
-func AMIRef(key string, fw FirmwareIdentity) *ebsmetadata.UEFIDataRef {
-	return &ebsmetadata.UEFIDataRef{
-		Key:      key,
-		Firmware: ebsmetadata.UEFIFirmware{Arch: fw.Arch, VarsSize: fw.VarsSize, CodeSHA256: fw.CodeSHA256},
-	}
-}
-
-// ReadObject gets key within GetTimeout and returns its decompressed region,
-// which must be exactly varsSize bytes, with the object's metadata. A missing
-// object is ErrObjectNotFound and a malformed one ErrInvalidObject.
-func ReadObject(ctx context.Context, objects objectstore.ObjectStore, bucket, key string, varsSize int64) ([]byte, map[string]*string, error) {
-	getCtx, cancel := context.WithTimeout(ctx, GetTimeout)
-	defer cancel()
-
-	out, err := objects.GetObject(getCtx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
-	if err != nil {
-		if objectstore.IsNoSuchKeyError(err) {
-			return nil, nil, fmt.Errorf("%w: %s", ErrObjectNotFound, key)
-		}
-		return nil, nil, fmt.Errorf("get UEFI store %s: %w", key, err)
-	}
-	defer out.Body.Close()
-
-	limit := varsSize + bodySlack
-	raw, err := io.ReadAll(io.LimitReader(out.Body, limit+1))
-	if err != nil {
-		return nil, nil, fmt.Errorf("read UEFI store %s: %w", key, err)
-	}
-	if int64(len(raw)) > limit {
-		return nil, nil, fmt.Errorf("%w: %s is over %d bytes compressed", ErrInvalidObject, key, limit)
-	}
-	gz, err := gzip.NewReader(bytes.NewReader(raw))
-	if err != nil {
-		return nil, nil, fmt.Errorf("%w: %s is not gzip: %w", ErrInvalidObject, key, err)
-	}
-	region, err := io.ReadAll(io.LimitReader(gz, varsSize+1))
-	if err != nil {
-		return nil, nil, fmt.Errorf("%w: %s does not decompress: %w", ErrInvalidObject, key, err)
-	}
-	if int64(len(region)) != varsSize {
-		return nil, nil, fmt.Errorf("%w: %s holds a %d-byte region, not the %d bytes its firmware needs",
-			ErrInvalidObject, key, len(region), varsSize)
-	}
-	return region, out.Metadata, nil
-}
-
-// CopyObject reads srcKey, a region of meta.Firmware's VARS size, and puts it
-// at a fresh key under meta.OwnerID, which it returns. A meta with no CodePath
-// takes the source object's.
-func CopyObject(ctx context.Context, objects objectstore.ObjectStore, bucket, srcKey string, meta ObjectMeta) (string, error) {
-	region, srcMeta, err := ReadObject(ctx, objects, bucket, srcKey, meta.Firmware.VarsSize)
-	if err != nil {
-		return "", err
-	}
-	if meta.CodePath == "" {
-		meta.CodePath = metadataValue(srcMeta, "firmware-code-path")
-	}
-	key, err := NewObjectKey(meta.OwnerID)
-	if err != nil {
-		return "", err
-	}
-	if err := PutObject(ctx, objects, bucket, key, region, meta); err != nil {
-		return "", err
-	}
-	return key, nil
-}
-
-// metadataValue reads name from S3 object metadata, whose keys the SDK
-// returns canonicalised.
-func metadataValue(md map[string]*string, name string) string {
-	for k, v := range md {
-		if strings.EqualFold(k, name) {
-			return aws.StringValue(v)
-		}
-	}
-	return ""
-}

spinifex/domains/ec2/instance/uefistore/image_test.go

@@ -1,70 +0,0 @@
-package uefistore_test
-
-import (
-	"testing"
-
-	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const imageID = "ami-0123456789abcdef0"
-
-// A launch only seeds from a store under the image's own prefix with a
-// firmware it can check; an AMI with no reference seeds from the template.
-func TestImageVarStoreFromAMI(t *testing.T) {
-	fw := ebsmetadata.UEFIFirmware{Arch: "x86_64", VarsSize: 4096, CodeSHA256: "c"}
-	own := uefistore.KeyPrefix(imageID) + "k"
-
-	got, err := uefistore.ImageVarStoreFromAMI(ebsmetadata.AMI{ImageID: imageID})
-	require.NoError(t, err)
-	assert.Nil(t, got)
-
-	got, err = uefistore.ImageVarStoreFromAMI(ebsmetadata.AMI{ImageID: imageID, UEFIData: &ebsmetadata.UEFIDataRef{Key: own, Firmware: fw}})
-	require.NoError(t, err)
-	assert.Equal(t, &uefistore.ImageVarStore{
-		ImageID: imageID, Key: own, Firmware: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: 4096, CodeSHA256: "c"},
-	}, got)
-
-	for name, ref := range map[string]ebsmetadata.UEFIDataRef{
-		"another image's key":   {Key: uefistore.KeyPrefix("ami-0fffffffffffffff0") + "k", Firmware: fw},
-		"an instance's key":     {Key: uefistore.KeyPrefix(instanceID) + "k", Firmware: fw},
-		"a nested key":          {Key: own + "/deeper", Firmware: fw},
-		"no firmware arch":      {Key: own, Firmware: ebsmetadata.UEFIFirmware{VarsSize: 4096}},
-		"no firmware VARS size": {Key: own, Firmware: ebsmetadata.UEFIFirmware{Arch: "x86_64"}},
-	} {
-		_, err := uefistore.ImageVarStoreFromAMI(ebsmetadata.AMI{ImageID: imageID, UEFIData: &ref})
-		require.ErrorIs(t, err, uefistore.ErrInvalidObject, name)
-	}
-}
-
-// A copy is a fresh key under the new owner holding the source's region, and
-// the source is left in place.
-func TestCopyObject_PutsTheRegionAtAFreshKeyUnderTheNewOwner(t *testing.T) {
-	objects := objectstore.NewMemoryObjectStore()
-	meta := testMeta(instanceID)
-	region := make([]byte, meta.Firmware.VarsSize)
-	copy(region, "variables")
-	src, err := uefistore.NewObjectKey(instanceID)
-	require.NoError(t, err)
-	require.NoError(t, uefistore.PutObject(t.Context(), objects, testBucket, src, region, meta))
-
-	dst := testMeta(imageID)
-	dst.Source = uefistore.SourceInstance(instanceID)
-	key, err := uefistore.CopyObject(t.Context(), objects, testBucket, src, dst)
-	require.NoError(t, err)
-
-	keys, err := uefistore.ListObjects(t.Context(), objects, testBucket, imageID)
-	require.NoError(t, err)
-	assert.Equal(t, []string{key}, keys)
-	got, _, err := uefistore.ReadObject(t.Context(), objects, testBucket, key, meta.Firmware.VarsSize)
-	require.NoError(t, err)
-	assert.Equal(t, region, got)
-	_, _, err = uefistore.ReadObject(t.Context(), objects, testBucket, src, meta.Firmware.VarsSize)
-	require.NoError(t, err, "the source stays")
-
-	_, err = uefistore.CopyObject(t.Context(), objects, testBucket, uefistore.KeyPrefix(instanceID)+"gone", dst)
-	require.ErrorIs(t, err, uefistore.ErrObjectNotFound)
-}

spinifex/domains/ec2/instance/uefistore/object.go

@@ -1,121 +0,0 @@
-package uefistore
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"crypto/rand"
-	"errors"
-	"fmt"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-)
-
-const (
-	// SourceTemplate is a store seeded from the launching node's VARS template.
-	SourceTemplate = "template"
-	// SourceWriteBack is a store written back from a node's live copy.
-	SourceWriteBack = "write-back"
-
-	// PutTimeout bounds one put of a store object. The object is a fixed-size
-	// region, so a put that takes longer is a wedged object store.
-	PutTimeout = 10 * time.Second
-)
-
-// ErrInvalidObject is a store object whose key, region or metadata cannot be
-// written as the owner's store.
-var ErrInvalidObject = errors.New("uefistore: invalid store object")
-
-// ObjectMeta is what a store object records about itself, as x-amz-meta-
-// entries, so its owner, writer and firmware are known from the object alone.
-type ObjectMeta struct {
-	OwnerID    string
-	WriterNode string
-	Firmware   FirmwareIdentity
-	CodePath   string
-	Source     string
-}
-
-// Metadata returns m as object metadata, keyed by the owner's kind.
-func (m ObjectMeta) Metadata() (map[string]*string, error) {
-	var ownerKey string
-	switch {
-	case strings.HasPrefix(m.OwnerID, "i-"):
-		ownerKey = "instance-id"
-	case strings.HasPrefix(m.OwnerID, "ami-"):
-		ownerKey = "image-id"
-	default:
-		return nil, fmt.Errorf("%w: owner %q is neither an instance nor an image", ErrInvalidObject, m.OwnerID)
-	}
-	switch {
-	case m.WriterNode == "":
-		return nil, fmt.Errorf("%w: %s names no writer node", ErrInvalidObject, m.OwnerID)
-	case m.Source == "":
-		return nil, fmt.Errorf("%w: %s names no source", ErrInvalidObject, m.OwnerID)
-	case m.Firmware.Arch == "" || m.Firmware.VarsSize <= 0 || m.Firmware.CodeSHA256 == "":
-		return nil, fmt.Errorf("%w: %s has an incomplete firmware identity", ErrInvalidObject, m.OwnerID)
-	}
-	return map[string]*string{
-		ownerKey:               aws.String(m.OwnerID),
-		"writer-node":          aws.String(m.WriterNode),
-		"firmware-arch":        aws.String(m.Firmware.Arch),
-		"firmware-vars-size":   aws.String(strconv.FormatInt(m.Firmware.VarsSize, 10)),
-		"firmware-code-sha256": aws.String(m.Firmware.CodeSHA256),
-		"firmware-code-path":   aws.String(m.CodePath),
-		"source":               aws.String(m.Source),
-	}, nil
-}
-
-// NewObjectKey returns a fresh key under ownerID's prefix, named by a random
-// version 4 UUID, so no write ever replaces an object a record points at.
-func NewObjectKey(ownerID string) (string, error) {
-	var b [16]byte
-	if _, err := rand.Read(b[:]); err != nil {
-		return "", fmt.Errorf("generate UEFI store key for %s: %w", ownerID, err)
-	}
-	b[6] = b[6]&0x0f | 0x40
-	b[8] = b[8]&0x3f | 0x80
-	return fmt.Sprintf("%s%x-%x-%x-%x-%x", KeyPrefix(ownerID), b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]), nil
-}
-
-// PutObject writes region gzip-compressed to key with meta, within
-// PutTimeout. The region must be exactly the firmware's VARS size, because
-// pflash refuses any other.
-func PutObject(ctx context.Context, objects objectstore.ObjectStore, bucket, key string, region []byte, meta ObjectMeta) error {
-	if !ownsKey(meta.OwnerID, key) {
-		return fmt.Errorf("%w: key %q is not under %s", ErrInvalidObject, key, KeyPrefix(meta.OwnerID))
-	}
-	if int64(len(region)) != meta.Firmware.VarsSize {
-		return fmt.Errorf("%w: %s is a %d-byte region under firmware with a %d-byte VARS region",
-			ErrInvalidObject, key, len(region), meta.Firmware.VarsSize)
-	}
-	md, err := meta.Metadata()
-	if err != nil {
-		return err
-	}
-	var body bytes.Buffer
-	zw := gzip.NewWriter(&body)
-	if _, err := zw.Write(region); err != nil {
-		return fmt.Errorf("compress UEFI store %s: %w", key, err)
-	}
-	if err := zw.Close(); err != nil {
-		return fmt.Errorf("compress UEFI store %s: %w", key, err)
-	}
-
-	putCtx, cancel := context.WithTimeout(ctx, PutTimeout)
-	defer cancel()
-	if _, err := objects.PutObject(putCtx, &s3.PutObjectInput{
-		Bucket:   aws.String(bucket),
-		Key:      aws.String(key),
-		Body:     bytes.NewReader(body.Bytes()),
-		Metadata: md,
-	}); err != nil {
-		return fmt.Errorf("put UEFI store %s: %w", key, err)
-	}
-	return nil
-}

spinifex/domains/ec2/instance/uefistore/object_test.go

@@ -1,164 +0,0 @@
-package uefistore_test
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"errors"
-	"io"
-	"regexp"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const testBucket = "instances"
-
-func testMeta(owner string) uefistore.ObjectMeta {
-	return uefistore.ObjectMeta{
-		OwnerID:    owner,
-		WriterNode: "node-1",
-		Firmware:   uefistore.FirmwareIdentity{Arch: "arm64", VarsSize: 1 << 16, CodeSHA256: "abc123"},
-		CodePath:   "/usr/share/AAVMF/AAVMF_CODE.fd",
-		Source:     uefistore.SourceTemplate,
-	}
-}
-
-var uuidV4 = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
-
-// Every key is a fresh version 4 UUID directly under the owner's prefix, so
-// a write never lands on an object a record already points at.
-func TestNewObjectKey_IsAFreshUUIDUnderTheOwnersPrefix(t *testing.T) {
-	seen := map[string]bool{}
-	for range 64 {
-		key, err := uefistore.NewObjectKey(instanceID)
-		require.NoError(t, err)
-		name, ok := bytes.CutPrefix([]byte(key), []byte(uefistore.KeyPrefix(instanceID)))
-		require.True(t, ok, key)
-		assert.Regexp(t, uuidV4, string(name))
-		assert.False(t, seen[key], "key %s repeated", key)
-		seen[key] = true
-	}
-}
-
-func TestObjectMeta_Metadata_NamesTheOwnerByKind(t *testing.T) {
-	md, err := testMeta(instanceID).Metadata()
-	require.NoError(t, err)
-	assert.Equal(t, map[string]string{
-		"instance-id":          instanceID,
-		"writer-node":          "node-1",
-		"firmware-arch":        "arm64",
-		"firmware-vars-size":   "65536",
-		"firmware-code-sha256": "abc123",
-		"firmware-code-path":   "/usr/share/AAVMF/AAVMF_CODE.fd",
-		"source":               "template",
-	}, aws.StringValueMap(md))
-
-	img := testMeta("ami-0123456789abcdef0")
-	img.Source = "image:ami-0fedcba9876543210"
-	md, err = img.Metadata()
-	require.NoError(t, err)
-	assert.Equal(t, "ami-0123456789abcdef0", aws.StringValue(md["image-id"]))
-	assert.NotContains(t, md, "instance-id")
-	assert.Equal(t, "image:ami-0fedcba9876543210", aws.StringValue(md["source"]))
-}
-
-func TestObjectMeta_Metadata_RefusesAnIncompleteObject(t *testing.T) {
-	tests := map[string]func(*uefistore.ObjectMeta){
-		"unknown owner":  func(m *uefistore.ObjectMeta) { m.OwnerID = "vol-1" },
-		"no writer":      func(m *uefistore.ObjectMeta) { m.WriterNode = "" },
-		"no source":      func(m *uefistore.ObjectMeta) { m.Source = "" },
-		"no arch":        func(m *uefistore.ObjectMeta) { m.Firmware.Arch = "" },
-		"no vars size":   func(m *uefistore.ObjectMeta) { m.Firmware.VarsSize = 0 },
-		"no code sha256": func(m *uefistore.ObjectMeta) { m.Firmware.CodeSHA256 = "" },
-	}
-	for name, mutate := range tests {
-		t.Run(name, func(t *testing.T) {
-			m := testMeta(instanceID)
-			mutate(&m)
-			_, err := m.Metadata()
-			require.ErrorIs(t, err, uefistore.ErrInvalidObject)
-		})
-	}
-}
-
-// The 64 MiB arm64 region is mostly erased flash, so it is stored compressed
-// and decompresses to the exact region.
-func TestPutObject_StoresTheRegionCompressed(t *testing.T) {
-	objects := objectstore.NewMemoryObjectStore()
-	meta := testMeta(instanceID)
-	region := bytes.Repeat([]byte{0xff}, int(meta.Firmware.VarsSize))
-	copy(region, "BootOrder")
-	key, err := uefistore.NewObjectKey(instanceID)
-	require.NoError(t, err)
-
-	require.NoError(t, uefistore.PutObject(t.Context(), objects, testBucket, key, region, meta))
-
-	out, err := objects.GetObject(t.Context(), &s3.GetObjectInput{Bucket: aws.String(testBucket), Key: aws.String(key)})
-	require.NoError(t, err)
-	body, err := io.ReadAll(out.Body)
-	require.NoError(t, err)
-	assert.Less(t, len(body), len(region)/10, "the erased region compresses")
-	gz, err := gzip.NewReader(bytes.NewReader(body))
-	require.NoError(t, err)
-	got, err := io.ReadAll(gz)
-	require.NoError(t, err)
-	assert.Equal(t, region, got)
-}
-
-// pflash needs a byte-exact region, and a key outside the owner's prefix is
-// one no store record owns; either is refused before anything is put.
-func TestPutObject_RefusesWhatTheOwnerCannotBoot(t *testing.T) {
-	meta := testMeta(instanceID)
-	good, err := uefistore.NewObjectKey(instanceID)
-	require.NoError(t, err)
-	other, err := uefistore.NewObjectKey("i-0fedcba9876543210")
-	require.NoError(t, err)
-	tests := []struct {
-		name   string
-		key    string
-		region []byte
-	}{
-		{name: "short region", key: good, region: make([]byte, meta.Firmware.VarsSize-1)},
-		{name: "long region", key: good, region: make([]byte, meta.Firmware.VarsSize+1)},
-		{name: "another owner's key", key: other, region: make([]byte, meta.Firmware.VarsSize)},
-		{name: "nested key", key: uefistore.KeyPrefix(instanceID) + "a/b", region: make([]byte, meta.Firmware.VarsSize)},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			objects := objectstore.NewMemoryObjectStore()
-			err := uefistore.PutObject(t.Context(), objects, testBucket, tt.key, tt.region, meta)
-			require.ErrorIs(t, err, uefistore.ErrInvalidObject)
-			out, listErr := objects.ListObjectsV2(t.Context(), &s3.ListObjectsV2Input{Bucket: aws.String(testBucket)})
-			require.NoError(t, listErr)
-			assert.Empty(t, out.Contents)
-		})
-	}
-}
-
-type failingPut struct {
-	*objectstore.MemoryObjectStore
-}
-
-func (failingPut) PutObject(ctx context.Context, _ *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
-	if _, ok := ctx.Deadline(); !ok {
-		return nil, errors.New("put had no deadline")
-	}
-	return nil, errors.New("predastore unreachable")
-}
-
-// A put runs under a deadline, so a wedged object store fails the write
-// rather than holding the caller, and its error reaches the caller.
-func TestPutObject_IsBoundedAndReportsAFailedPut(t *testing.T) {
-	key, err := uefistore.NewObjectKey(instanceID)
-	require.NoError(t, err)
-	meta := testMeta(instanceID)
-	err = uefistore.PutObject(t.Context(), failingPut{objectstore.NewMemoryObjectStore()}, testBucket, key, make([]byte, meta.Firmware.VarsSize), meta)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "predastore unreachable")
-}

spinifex/domains/ec2/instance/uefistore/prune.go

@@ -1,73 +0,0 @@
-package uefistore
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-)
-
-// PruneMinAge is how old an object must be before a prune may delete it, so a
-// write-back whose pointer move has not landed yet never loses its object.
-const PruneMinAge = 10 * time.Minute
-
-// PruneResult is what one prune of a prefix did.
-type PruneResult struct {
-	Deleted int
-	// Young counts objects other than keep that were too new to delete, so the
-	// caller prunes the prefix again later.
-	Young int
-}
-
-// PruneObjects deletes every object under ownerID's prefix except keep that was
-// last modified before cutoff. keep must be the key the store record points at;
-// an empty or foreign keep deletes nothing. An object with no modification time
-// counts as young.
-func PruneObjects(ctx context.Context, objects objectstore.ObjectStore, bucket, ownerID, keep string, cutoff time.Time) (PruneResult, error) {
-	var res PruneResult
-	if err := checkOwnedKey(ownerID, keep); err != nil {
-		return res, fmt.Errorf("uefistore: refusing to prune %s: %w", ownerID, err)
-	}
-	contents, _, err := objectstore.ListAll(ctx, objects, &s3.ListObjectsV2Input{
-		Bucket: aws.String(bucket),
-		Prefix: aws.String(KeyPrefix(ownerID)),
-	})
-	if err != nil {
-		return res, fmt.Errorf("uefistore: list %s: %w", KeyPrefix(ownerID), err)
-	}
-	for _, obj := range contents {
-		key := aws.StringValue(obj.Key)
-		if key == keep {
-			continue
-		}
-		if obj.LastModified == nil || !obj.LastModified.Before(cutoff) {
-			res.Young++
-			continue
-		}
-		_, err := objects.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
-		if err != nil && !objectstore.IsNoSuchKeyError(err) {
-			return res, fmt.Errorf("uefistore: delete %s: %w", key, err)
-		}
-		res.Deleted++
-	}
-	return res, nil
-}
-
-// checkOwnedKey fails a key that is empty or not one object under ownerID's
-// prefix.
-func checkOwnedKey(ownerID, key string) error {
-	if !validOwner(ownerID) {
-		return fmt.Errorf("%q is not an instance or image ID", ownerID)
-	}
-	if key == "" {
-		return errors.New("no key to keep")
-	}
-	if !ownsKey(ownerID, key) {
-		return fmt.Errorf("%s is not under %s", key, KeyPrefix(ownerID))
-	}
-	return nil
-}

spinifex/domains/ec2/instance/uefistore/prune_test.go

@@ -1,84 +0,0 @@
-package uefistore_test
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// datedStore lists each object with the modification time the test set; a
-// key it has none for is listed with none.
-type datedStore struct {
-	*objectstore.MemoryObjectStore
-
-	modified map[string]time.Time
-}
-
-func (s *datedStore) ListObjectsV2(ctx context.Context, in *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
-	out, err := s.MemoryObjectStore.ListObjectsV2(ctx, in)
-	if err != nil {
-		return nil, err
-	}
-	for _, obj := range out.Contents {
-		obj.LastModified = nil
-		if at, ok := s.modified[aws.StringValue(obj.Key)]; ok {
-			obj.LastModified = aws.Time(at)
-		}
-	}
-	return out, nil
-}
-
-// A prune keeps the key it is told to and every object not older than the
-// cutoff, counting those it kept for age, and an undated object counts as
-// young (ADR-0003:S7).
-func TestS7_PruneObjectsKeepsThePointerAndEveryYoungObject(t *testing.T) {
-	const owner = "i-0123456789abcdef0"
-	prefix := uefistore.KeyPrefix(owner)
-	cutoff := time.Unix(1700000000, 0)
-	keep, stale, young, undated, other := prefix+"keep", prefix+"stale", prefix+"young", prefix+"undated", uefistore.KeyPrefix("i-0fedcba9876543210")+"stale"
-	store := &datedStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), modified: map[string]time.Time{
-		keep:  cutoff.Add(-time.Hour),
-		stale: cutoff.Add(-time.Second),
-		young: cutoff,
-		other: cutoff.Add(-time.Hour),
-	}}
-	for _, k := range []string{keep, stale, young, undated, other} {
-		putObject(t, store, k)
-	}
-
-	res, err := uefistore.PruneObjects(t.Context(), store, bucket, owner, keep, cutoff)
-
-	require.NoError(t, err)
-	assert.Equal(t, uefistore.PruneResult{Deleted: 1, Young: 2}, res)
-	assert.ElementsMatch(t, []string{keep, young, undated}, keysUnder(t, store, prefix),
-		"ADR-0003:S7: a prune must keep the pointed object and every object younger than the cutoff")
-	assert.Equal(t, []string{other}, keysUnder(t, store, uefistore.KeyPrefix("i-0fedcba9876543210")), "another owner's prefix is untouched")
-}
-
-// With no key to keep, or one outside the owner's prefix, a prune cannot tell
-// the pointer from a leftover, so it deletes nothing (ADR-0003:S7).
-func TestS7_PruneObjectsWithoutAKeyToKeepDeletesNothing(t *testing.T) {
-	const owner = "i-0123456789abcdef0"
-	prefix := uefistore.KeyPrefix(owner)
-	store := &datedStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), modified: map[string]time.Time{
-		prefix + "stale": time.Unix(1, 0),
-	}}
-	putObject(t, store, prefix+"stale")
-
-	for _, keep := range []string{"", uefistore.KeyPrefix("i-0fedcba9876543210") + "stale", prefix + "nested/key"} {
-		_, err := uefistore.PruneObjects(t.Context(), store, bucket, owner, keep, time.Now())
-		require.Error(t, err, keep)
-	}
-	_, err := uefistore.PruneObjects(t.Context(), store, bucket, "i-*", prefix+"stale", time.Now())
-	require.Error(t, err)
-
-	assert.Equal(t, []string{prefix + "stale"}, keysUnder(t, store, prefix),
-		"ADR-0003:S7: a prune with no valid pointer to keep must delete nothing")
-}

spinifex/domains/ec2/instance/uefistore/uefistore.go

@@ -1,6 +1,6 @@
 // Package uefistore holds the types that describe an instance's UEFI variable
-// store, the bucket of store records that carry its pointer and deletion
-// obligation past the life of the instance record, and the claim that fences it.
+// store and the bucket of store records that carry its deletion and write-back
+// obligations past the life of the instance record.
 package uefistore
 
 import (
@@ -53,27 +53,22 @@ type FirmwareIdentity struct {
 // UEFIStoreRecord is the durable owner of one instance's variable store
 // objects. It outlives the instance record, so an obligation recorded here is
 // not lost when a terminated record expires.
-//
-// Pointer names the only version of the store the record keeps, the object the
-// instance boots from, and only a writer whose claim token has generation
-// Generation may move it.
 type UEFIStoreRecord struct {
-	SchemaVersion uint16           `json:"schema_version"`
-	InstanceID    string           `json:"instance_id"`
-	AccountID     string           `json:"account_id"`
-	State         string           `json:"state"`
-	Generation    uint64           `json:"generation"`
-	Pointer       *UEFIVarStoreRef `json:"pointer,omitempty"`
+	SchemaVersion uint16         `json:"schema_version"`
+	InstanceID    string         `json:"instance_id"`
+	AccountID     string         `json:"account_id"`
+	State         string         `json:"state"`
+	WriteBack     *UEFIWriteBack `json:"write_back,omitempty"`
 }
 
-// UEFIVarStoreRef names the store object an instance launches from, with the
-// firmware it was written under, so a launch can check it without reading the
-// object's metadata.
-type UEFIVarStoreRef struct {
-	Key        string           `json:"key"`
-	Firmware   FirmwareIdentity `json:"firmware"`
-	WriterNode string           `json:"writer_node"`
-	WrittenAt  time.Time        `json:"written_at"`
+// UEFIWriteBack is a write-back that failed and is still owed. Key is the
+// object every retry puts to, so retries leave no extra objects; SeededFrom is
+// the pointer Node seeded from, which a retry must still find on the record.
+type UEFIWriteBack struct {
+	Node       string    `json:"node"`
+	Key        string    `json:"key"`
+	SeededFrom string    `json:"seeded_from"`
+	Since      time.Time `json:"since"`
 }
 
 // KeyPrefix is the object-store prefix every store object of ownerID (an
@@ -88,8 +83,8 @@ func ownsKey(ownerID, key string) bool {
 	return ok && name != "" && !strings.Contains(name, "/")
 }
 
-// validate checks rec against key. The pointer must name one object directly
-// under the instance's own prefix, because a deletion acts on it.
+// validate checks rec against key. A write-back key outside the instance's own
+// prefix is refused because a superseded obligation deletes that key.
 func (rec *UEFIStoreRecord) validate(key string) error {
 	switch {
 	case rec.SchemaVersion > RecordSchemaVersion:
@@ -103,32 +98,29 @@ func (rec *UEFIStoreRecord) validate(key string) error {
 		return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
 	case rec.State != StateLive && rec.State != StateDeleting:
 		return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
+	case rec.State == StateDeleting && rec.WriteBack != nil:
+		// A write-back retry puts an object, which would land behind the delete.
+		return fmt.Errorf("%w: %s is deleting with a write-back still owed", ErrInvalidRecord, key)
 	}
-	if p := rec.Pointer; p != nil {
-		if err := p.validate(rec.InstanceID); err != nil {
-			return fmt.Errorf("%w: %s pointer: %w", ErrInvalidRecord, key, err)
+	if wb := rec.WriteBack; wb != nil {
+		switch {
+		case wb.Node == "":
+			return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
+		case wb.Since.IsZero():
+			return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
+		case !ownsKey(rec.InstanceID, wb.Key):
+			return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
+		case !ownsKey(rec.InstanceID, wb.SeededFrom):
+			return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
 		}
 	}
 	return nil
 }
 
-// validate checks ref as a pointer of instanceID's store.
-func (ref *UEFIVarStoreRef) validate(instanceID string) error {
-	switch {
-	case !ownsKey(instanceID, ref.Key):
-		return fmt.Errorf("key %q is not under %s", ref.Key, KeyPrefix(instanceID))
-	case ref.Firmware.Arch == "" || ref.Firmware.VarsSize <= 0:
-		return fmt.Errorf("key %q names no firmware arch and VARS size", ref.Key)
-	case ref.WriterNode == "":
-		return fmt.Errorf("key %q names no writer node", ref.Key)
-	}
-	return nil
-}
-
 func bucketConfig() kvstore.Config {
 	return kvstore.Config{
 		Name:        BucketName,
-		Description: "Spinifex UEFI variable store records (pointers and deletion obligations)",
+		Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
 		History:     1,
 		// Left false: a lost bucket has lost obligations, and an empty
 		// recreated one would report every store as having none.
@@ -180,19 +172,6 @@ func (s *Store) Get(ctx context.Context, instanceID string) (*UEFIStoreRecord, u
 	return rec, rev, nil
 }
 
-// LeaderGet is Get answered by the stream leader, for a caller that deletes
-// objects by what the record does not point at.
-func (s *Store) LeaderGet(ctx context.Context, instanceID string) (*UEFIStoreRecord, uint64, error) {
-	rec, rev, err := s.kv.LeaderGet(ctx, instanceID)
-	if err != nil {
-		return nil, 0, err
-	}
-	if err := rec.validate(instanceID); err != nil {
-		return nil, 0, err
-	}
-	return rec, rev, nil
-}
-
 // Create writes a new record, which must be live: a store is recorded before
 // its first object is written. An existing record is kvstore.ErrExists. Every
 // write stamps rec with RecordSchemaVersion.

spinifex/domains/ec2/instance/uefistore/uefistore_test.go

@@ -11,6 +11,7 @@ import (
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -30,25 +31,15 @@ func liveRecord() *uefistore.UEFIStoreRecord {
 	return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
 }
 
-func pointer(name string) *uefistore.UEFIVarStoreRef {
-	return &uefistore.UEFIVarStoreRef{
-		Key:        keyOf(name),
-		Firmware:   uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: 540672, CodeSHA256: "c0de"},
-		WriterNode: "node-2",
-		WrittenAt:  time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
+func writeBack() *uefistore.UEFIWriteBack {
+	return &uefistore.UEFIWriteBack{
+		Node:       "node-2",
+		Key:        uefistore.KeyPrefix(instanceID) + "b",
+		SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
+		Since:      time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
 	}
 }
 
-func keyOf(name string) string { return uefistore.KeyPrefix(instanceID) + name }
-
-// pointedRecord is a live record stamped at generation 7 that points at "b".
-func pointedRecord() *uefistore.UEFIStoreRecord {
-	rec := liveRecord()
-	rec.Generation = 7
-	rec.Pointer = pointer("b")
-	return rec
-}
-
 func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
 	t.Helper()
 	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
@@ -69,14 +60,15 @@ func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
 	assert.Equal(t, int64(1), status.History())
 	assert.Zero(t, status.TTL(), "a TTL would expire deletion obligations before they are settled")
 
-	v, err := kvstore.ReadVersion(t.Context(), kv)
+	v, err := kvutil.ReadVersion(t.Context(), kv)
 	require.NoError(t, err)
 	assert.Equal(t, uefistore.BucketVersion, v)
 }
 
 func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
 	s, _ := openStore(t)
-	rec := pointedRecord()
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
 
 	rev, err := s.Create(t.Context(), rec)
 	require.NoError(t, err)
@@ -91,7 +83,9 @@ func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
 // The wire names are what every release reads, so they are pinned.
 func TestRecord_WireShape(t *testing.T) {
 	s, js := openStore(t)
-	_, err := s.Create(t.Context(), pointedRecord())
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+	_, err := s.Create(t.Context(), rec)
 	require.NoError(t, err)
 
 	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
@@ -103,12 +97,11 @@ func TestRecord_WireShape(t *testing.T) {
 		"instance_id": "i-0123456789abcdef0",
 		"account_id": "111122223333",
 		"state": "live",
-		"generation": 7,
-		"pointer": {
+		"write_back": {
+			"node": "node-2",
 			"key": "uefi/i-0123456789abcdef0/b",
-			"firmware": {"arch": "x86_64", "vars_size": 540672, "code_sha256": "c0de"},
-			"writer_node": "node-2",
-			"written_at": "2026-11-03T01:02:03Z"
+			"seeded_from": "uefi/i-0123456789abcdef0/a",
+			"since": "2026-11-03T01:02:03Z"
 		}
 	}`, string(entry.Value()))
 }
@@ -152,7 +145,8 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
 	first, err := s.Create(t.Context(), liveRecord())
 	require.NoError(t, err)
 
-	moved := pointedRecord()
+	moved := liveRecord()
+	moved.WriteBack = writeBack()
 	second, err := s.CompareAndSet(t.Context(), moved, first)
 	require.NoError(t, err)
 	assert.Greater(t, second, first)
@@ -166,7 +160,7 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, second, rev)
 	assert.Equal(t, uefistore.StateLive, got.State)
-	assert.Equal(t, pointer("b"), got.Pointer)
+	assert.Equal(t, writeBack(), got.WriteBack)
 }
 
 // Two actors that read the same revision race to change it; exactly one may
@@ -186,11 +180,12 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
 	)
 	for i := range writers {
 		wg.Go(func() {
-			rec := pointedRecord()
-			rec.Pointer.WriterNode = fmt.Sprintf("node-%d", i)
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
+			rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
 			if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
 				mu.Lock()
-				wins = append(wins, rec.Pointer.WriterNode)
+				wins = append(wins, rec.WriteBack.Node)
 				mu.Unlock()
 			} else {
 				assert.ErrorIs(t, err, kvstore.ErrConflict)
@@ -202,7 +197,7 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
 	require.Len(t, wins, 1)
 	got, _, err := s.Get(t.Context(), instanceID)
 	require.NoError(t, err)
-	assert.Equal(t, wins[0], got.Pointer.WriterNode)
+	assert.Equal(t, wins[0], got.WriteBack.Node)
 }
 
 func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
@@ -223,20 +218,29 @@ func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
 	require.ErrorIs(t, err, kvstore.ErrNotFound)
 }
 
-// A deletion acts on the key the record points at, so a key outside the
+// A superseded obligation deletes its recorded key, so a key outside the
 // instance's own prefix must never reach the bucket.
 func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
-	other := uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
 	cases := map[string]func(*uefistore.UEFIStoreRecord){
-		"unknown state":                   func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
-		"not an instance ID":              func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
-		"pointer under another instance":  func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = other },
-		"pointer under an image":          func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b" },
-		"pointer is the prefix itself":    func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix(instanceID) },
-		"pointer nested below the prefix": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = keyOf("x/y") },
-		"pointer with no firmware arch":   func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.Arch = "" },
-		"pointer with no VARS size":       func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.VarsSize = 0 },
-		"pointer with no writer node":     func(r *uefistore.UEFIStoreRecord) { r.Pointer.WriterNode = "" },
+		"unknown state":            func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
+		"not an instance ID":       func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
+		"write-back with no node":  func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
+		"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
+		"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
+		},
+		"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
+		},
+		"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
+		},
+		"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
+		},
+		"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
+		},
 	}
 	for name, mutate := range cases {
 		t.Run(name, func(t *testing.T) {
@@ -244,7 +248,8 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 			rev, err := s.Create(t.Context(), liveRecord())
 			require.NoError(t, err)
 
-			rec := pointedRecord()
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
 			mutate(rec)
 			_, err = s.CompareAndSet(t.Context(), rec, rev)
 			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
@@ -252,7 +257,7 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 			got, gotRev, err := s.Get(t.Context(), instanceID)
 			require.NoError(t, err)
 			assert.Equal(t, rev, gotRev, "an invalid record must not be written")
-			assert.Nil(t, got.Pointer)
+			assert.Nil(t, got.WriteBack)
 		})
 	}
 }
@@ -261,11 +266,11 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 // is reported rather than decoded into something a reaper would act on.
 func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
 	cases := map[string]string{
-		"newer schema":     `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
-		"no schema":        `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
-		"another instance": `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
-		"unknown state":    `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
-		"foreign pointer":  `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","generation":3,"pointer":{"key":"uefi/i-0fedcba9876543210/b","firmware":{"arch":"x86_64","vars_size":540672},"writer_node":"n"}}`,
+		"newer schema":       `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"no schema":          `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"another instance":   `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
+		"unknown state":      `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
+		"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
 	}
 	for name, body := range cases {
 		t.Run(name, func(t *testing.T) {

spinifex/domains/ec2/instance/volume_held_elsewhere_test.go

@@ -102,38 +102,3 @@ func TestStartPaths_ReportAVolumeHeldElsewhereByCode(t *testing.T) {
 		assert.Equal(t, awserrors.ErrorIncorrectState, awserrors.ValidErrorCodeFromError(err))
 	})
 }
-
-// uefiClaimHeldMounter fails the launch the way a start does while a live
-// node holds the instance's UEFI variable store.
-type uefiClaimHeldMounter struct{ raceVolumeMounter }
-
-func (uefiClaimHeldMounter) Mount(context.Context, *vm.VM) error {
-	return fmt.Errorf("%w: node-a holds i-uefi: %w", vm.ErrUEFIClaimHeld, vm.ErrMountRetryable)
-}
-
-// A held UEFI store is a capacity refusal on both start paths, so the caller
-// tries another node rather than reporting a fault.
-func TestStartPaths_ReportAHeldUEFIStoreAsCapacity(t *testing.T) {
-	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
-	types, _ := defaultPrepareInstanceTypes()
-
-	t.Run("claimed relaunch", func(t *testing.T) {
-		mgr := vm.NewManager()
-		mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
-		svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
-
-		err := svc.launchClaimedInstance(t.Context(), &vm.VM{ID: "i-uefi-claim", AccountID: "acc", InstanceType: "t3.micro", Status: vm.StatePending}, func() {})
-		assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
-	})
-
-	t.Run("live owner", func(t *testing.T) {
-		const id = "i-uefi-live"
-		mgr := mgrWith(map[string]*vm.VM{id: {ID: id, Status: vm.StateStopped, InstanceType: "t3.micro"}})
-		mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
-		v, _ := mgr.Get(id)
-		svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
-
-		err := svc.StartInstance(t.Context(), v, ec2v1.EC2InstanceCommand{ID: id})
-		assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
-	})
-}

spinifex/domains/ec2/launchtemplate/account_cleanup.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -33,7 +33,7 @@ type AccountCleanup struct {
 // Unreadable headers, their bodies and unrecognised entries are left for the
 // owner scan to report. A listing failure is returned and nothing is removed.
 func (s *LaunchTemplateServiceImpl) CleanupAccountResidue(ctx context.Context, accountID string, settledBefore time.Time) (*AccountCleanup, error) {
-	keys, err := kvstore.Keys(ctx, s.kv)
+	keys, err := kvutil.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list launch-template keys: %w", err)
 	}
@@ -145,7 +145,7 @@ type AccountTemplateDeletes struct {
 // it cannot read or decode is left for the owner scan to report. A listing
 // failure is returned and nothing is deleted.
 func (s *LaunchTemplateServiceImpl) DeleteLiveTemplates(ctx context.Context, accountID string, referenced func(id, name string) string) (*AccountTemplateDeletes, error) {
-	keys, err := kvstore.Keys(ctx, s.kv)
+	keys, err := kvutil.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list launch-template keys: %w", err)
 	}

spinifex/domains/ec2/launchtemplate/account_scan.go

@@ -9,7 +9,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -37,7 +37,7 @@ type AccountScan struct {
 // Headers and name-index entries are read; version bodies are classified from
 // their keys alone, so template data and user data are never loaded.
 func (s *LaunchTemplateServiceImpl) ScanAccount(ctx context.Context, accountID string) (*AccountScan, error) {
-	keys, err := kvstore.Keys(ctx, s.kv)
+	keys, err := kvutil.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list launch-template keys: %w", err)
 	}

spinifex/domains/ec2/launchtemplate/account_scan_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit/readonlystore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -45,7 +45,7 @@ func TestScanAccountClassifiesLaunchTemplateRecords(t *testing.T) {
 	// A header removed alone, as a delete interrupted after it leaves the rest.
 	deleted := createWithUserData(t, svc, "deleted")
 	inner := svc.kv
-	require.NoError(t, inner.Delete(t.Context(), kvstore.AccountKey(testAccountID, deleted)))
+	require.NoError(t, inner.Delete(t.Context(), kvutil.AccountKey(testAccountID, deleted)))
 
 	// A create whose header write and clean-up both fail.
 	svc.kv = &faultyKV{KeyValue: inner,
@@ -59,16 +59,16 @@ func TestScanAccountClassifiesLaunchTemplateRecords(t *testing.T) {
 	require.Error(t, err)
 	svc.kv = inner
 
-	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0badbadbadbadbad0"), "{not json")
-	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0badbadbadbadbad0.v1"), `{"version_number":1}`)
-	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0mismatch00000000"), `{"launch_template_id":"lt-0other"}`)
-	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0unreadable000000"), `{"launch_template_id":"lt-0unreadable000000"}`)
+	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0badbadbadbadbad0"), "{not json")
+	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0badbadbadbadbad0.v1"), `{"version_number":1}`)
+	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0mismatch00000000"), `{"launch_template_id":"lt-0other"}`)
+	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0unreadable000000"), `{"launch_template_id":"lt-0unreadable000000"}`)
 	putRaw(t, inner, nameKey(testAccountID, "unreadable"), "lt-0unreadable000000")
-	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0bad.vx"), "{}")
+	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0bad.vx"), "{}")
 
 	recorder := &readRecordingKV{KeyValue: readonlystore.NewKV(t, inner)}
 	svc.kv = &faultyKV{KeyValue: recorder, failGet: func(key string) bool {
-		return key == kvstore.AccountKey(testAccountID, "lt-0unreadable000000")
+		return key == kvutil.AccountKey(testAccountID, "lt-0unreadable000000")
 	}}
 	scan, err := svc.ScanAccount(context.Background(), testAccountID)
 	require.NoError(t, err)

spinifex/domains/ec2/launchtemplate/account_termination_current_behaviour_test.go

@@ -16,7 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -159,7 +159,7 @@ func TestCurrentBehaviour_AccountTermination_LaunchTemplateStorageLayout(t *test
 
 	header := testAccountID + "." + id
 	name := testAccountID + ".name.776562283129" // hex of "web(1)"
-	assert.Equal(t, sortedKeys(header, header+".v1", header+".v2", name, kvstore.VersionKey), storedKeys(t, svc.kv))
+	assert.Equal(t, sortedKeys(header, header+".v1", header+".v2", name, kvutil.VersionKey), storedKeys(t, svc.kv))
 
 	entry, err := svc.kv.Get(t.Context(), name)
 	require.NoError(t, err)
@@ -223,13 +223,13 @@ func TestAccountTermination_DeleteLaunchTemplateKeepsTheTemplateWhenAStepFails(t
 			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 
 			svc.kv = inner
-			assert.Equal(t, sortedKeys(append(tc.left(header, nameIdx), kvstore.VersionKey)...), storedKeys(t, inner))
+			assert.Equal(t, sortedKeys(append(tc.left(header, nameIdx), kvutil.VersionKey)...), storedKeys(t, inner))
 			assert.Equal(t, []string{id}, describedTemplateIDs(t, svc, testAccountID))
 
 			out, err := svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)
 			require.NoError(t, err)
 			assert.Equal(t, id, aws.StringValue(out.LaunchTemplate.LaunchTemplateId))
-			assert.Equal(t, []string{kvstore.VersionKey}, storedKeys(t, inner))
+			assert.Equal(t, []string{kvutil.VersionKey}, storedKeys(t, inner))
 			assert.Empty(t, describedTemplateIDs(t, svc, testAccountID))
 		})
 	}
@@ -260,7 +260,7 @@ func TestAccountTermination_DeleteLaunchTemplateRestartsWhenTheHeaderChanges(t *
 	_, err = svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)
 	require.NoError(t, err)
 	assert.True(t, raced)
-	assert.Equal(t, []string{kvstore.VersionKey}, storedKeys(t, inner))
+	assert.Equal(t, []string{kvutil.VersionKey}, storedKeys(t, inner))
 }
 
 // A create that fails after claiming the name removes what it wrote, so no
@@ -287,7 +287,7 @@ func TestAccountTermination_FailedCreateRemovesWhatItWrote(t *testing.T) {
 			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 
 			svc.kv = inner
-			assert.Equal(t, []string{kvstore.VersionKey}, storedKeys(t, inner))
+			assert.Equal(t, []string{kvutil.VersionKey}, storedKeys(t, inner))
 			retried := createTemplate(t, svc, "web", "t3.micro")
 			assert.Equal(t, []string{aws.StringValue(retried.LaunchTemplateId)}, describedTemplateIDs(t, svc, testAccountID))
 		})
@@ -299,7 +299,7 @@ func TestAccountTermination_FailedCreateRemovesWhatItWrote(t *testing.T) {
 // failure is an internal error, never a missing template.
 func TestCurrentBehaviour_AccountTermination_DescribeLaunchTemplatesHidesUnreadableHeaders(t *testing.T) {
 	const brokenID = "lt-0badbadbadbadbad0"
-	brokenHeader := kvstore.AccountKey(testAccountID, brokenID)
+	brokenHeader := kvutil.AccountKey(testAccountID, brokenID)
 	tests := []struct {
 		name       string
 		seed       func(t *testing.T, svc *LaunchTemplateServiceImpl)
@@ -350,7 +350,7 @@ func TestAccountCleanup_KeepsTemplatesAndRemovesSettledResidue(t *testing.T) {
 	require.NoError(t, err)
 	orphanBody := versionKey(testAccountID, "lt-0orphan000000000", 1)
 	orphanName := nameKey(testAccountID, "gone")
-	brokenHeader := kvstore.AccountKey(testAccountID, "lt-0broken000000000")
+	brokenHeader := kvutil.AccountKey(testAccountID, "lt-0broken000000000")
 	brokenBody := versionKey(testAccountID, "lt-0broken000000000", 1)
 	brokenName := nameKey(testAccountID, "broken")
 	for key, value := range map[string]string{
@@ -384,7 +384,7 @@ func TestAccountCleanup_KeepsTemplatesAndRemovesSettledResidue(t *testing.T) {
 func TestAccountCleanup_InterruptedTemplateDeleteIsLeftToThePublicRetry(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
-	header := kvstore.AccountKey(testAccountID, id)
+	header := kvutil.AccountKey(testAccountID, id)
 	kv := svc.kv
 	svc.kv = &faultyKV{KeyValue: kv, failDelete: func(key string) bool { return key == header }}
 	_, err := svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)

spinifex/domains/ec2/launchtemplate/service_impl.go

@@ -19,7 +19,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -64,7 +64,7 @@ func NewLaunchTemplateServiceImplWithNATS(ctx context.Context, cfg *config.Confi
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketLaunchTemplates, 10)
+	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketLaunchTemplates, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketLaunchTemplates, err)
 	}
@@ -87,24 +87,24 @@ func NewLaunchTemplateServiceImplWithNATS(ctx context.Context, cfg *config.Confi
 // Launch template names allow characters such as parentheses that are not valid
 // in NATS KV keys, so the name must not be used in its raw form here.
 func nameKey(accountID, name string) string {
-	return kvstore.AccountKey(accountID, "name."+hex.EncodeToString([]byte(name)))
+	return kvutil.AccountKey(accountID, "name."+hex.EncodeToString([]byte(name)))
 }
 
 // versionKey returns a version-body key: account.lt-<id>.v<n>.
 func versionKey(accountID, ltID string, n int64) string {
-	return kvstore.AccountKey(accountID, ltID+".v"+strconv.FormatInt(n, 10))
+	return kvutil.AccountKey(accountID, ltID+".v"+strconv.FormatInt(n, 10))
 }
 
 // versionPrefix returns the scan prefix for a template's version bodies.
 func versionPrefix(accountID, ltID string) string {
-	return kvstore.AccountKey(accountID, ltID+".v")
+	return kvutil.AccountKey(accountID, ltID+".v")
 }
 
 // --- record load helpers ---
 
 // getHeaderByID reads a header and its KV entry (for CAS) by launch template id.
 func (s *LaunchTemplateServiceImpl) getHeaderByID(ctx context.Context, accountID, ltID string) (*LaunchTemplateHeader, jetstream.KeyValueEntry, error) {
-	entry, err := s.kv.Get(ctx, kvstore.AccountKey(accountID, ltID))
+	entry, err := s.kv.Get(ctx, kvutil.AccountKey(accountID, ltID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
 			return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdNotFound)
@@ -386,7 +386,7 @@ func (s *LaunchTemplateServiceImpl) claimName(ctx context.Context, accountID, na
 	// header means the name is taken; any other read error (transient fault, or a
 	// concurrent in-flight create whose header is not yet written) fails closed so
 	// the name is never stolen from a possibly-live template.
-	_, herr := s.kv.Get(ctx, kvstore.AccountKey(accountID, string(entry.Value())))
+	_, herr := s.kv.Get(ctx, kvutil.AccountKey(accountID, string(entry.Value())))
 	switch {
 	case herr == nil:
 		return errNameInUse()
@@ -404,7 +404,7 @@ func (s *LaunchTemplateServiceImpl) putHeader(ctx context.Context, accountID str
 	if err != nil {
 		return errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.kv.Put(ctx, kvstore.AccountKey(accountID, h.LaunchTemplateId), data); err != nil {
+	if _, err := s.kv.Put(ctx, kvutil.AccountKey(accountID, h.LaunchTemplateId), data); err != nil {
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 	return nil
@@ -510,13 +510,13 @@ func (s *LaunchTemplateServiceImpl) commitVersion(ctx context.Context, accountID
 // re-reading on a conflict. A non-nil check vets each re-read header and
 // stops the commit with its error.
 func (s *LaunchTemplateServiceImpl) commitHeader(ctx context.Context, op, accountID, ltID string, entry jetstream.KeyValueEntry, check func(*LaunchTemplateHeader) error) error {
-	key := kvstore.AccountKey(accountID, ltID)
+	key := kvutil.AccountKey(accountID, ltID)
 	for range maxVersionRetries {
 		_, err := s.kv.Update(ctx, key, entry.Value(), entry.Revision())
 		if err == nil {
 			return nil
 		}
-		if !kvstore.IsRevisionConflict(err) {
+		if !kvutil.IsRevisionConflict(err) {
 			slog.ErrorContext(ctx, op+": header commit failed", "launchTemplateId", ltID, "err", err)
 			return err
 		}
@@ -590,7 +590,7 @@ func (s *LaunchTemplateServiceImpl) ModifyLaunchTemplate(ctx context.Context, in
 // header deleted meanwhile is NotFound for the identifier the request used.
 func (s *LaunchTemplateServiceImpl) setDefaultVersion(ctx context.Context, accountID string, input *ec2.ModifyLaunchTemplateInput, header *LaunchTemplateHeader, entry jetstream.KeyValueEntry, sel string) (*LaunchTemplateHeader, error) {
 	notFound := fmt.Sprintf("The launch template version %s is not found for the specified launch template.", sel)
-	key := kvstore.AccountKey(accountID, header.LaunchTemplateId)
+	key := kvutil.AccountKey(accountID, header.LaunchTemplateId)
 	for range maxVersionRetries {
 		n, err := s.resolveVersionNumber(ctx, accountID, header, sel)
 		if err != nil {
@@ -611,7 +611,7 @@ func (s *LaunchTemplateServiceImpl) setDefaultVersion(ctx context.Context, accou
 		if err == nil {
 			return header, nil
 		}
-		if !kvstore.IsRevisionConflict(err) {
+		if !kvutil.IsRevisionConflict(err) {
 			slog.ErrorContext(ctx, "ModifyLaunchTemplate: header update failed", "launchTemplateId", header.LaunchTemplateId, "err", err)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
@@ -662,7 +662,7 @@ const maxHeaderDeleteAttempts = 3
 // through the public delete; a header changed in between restarts the sweep.
 func (s *LaunchTemplateServiceImpl) deleteTemplate(ctx context.Context, accountID string, header *LaunchTemplateHeader, entry jetstream.KeyValueEntry) error {
 	ltID := header.LaunchTemplateId
-	headerKey := kvstore.AccountKey(accountID, ltID)
+	headerKey := kvutil.AccountKey(accountID, ltID)
 	for attempt := 1; ; attempt++ {
 		if err := s.deleteVersionBodies(ctx, accountID, ltID); err != nil {
 			return err
@@ -824,7 +824,7 @@ func (s *LaunchTemplateServiceImpl) commitVersionDeletes(ctx context.Context, ac
 	}
 	// A template delete that removed the header before the restore has already
 	// swept its bodies, so the restored ones would be left headerless.
-	if _, gerr := s.kv.Get(ctx, kvstore.AccountKey(accountID, ltID)); errors.Is(gerr, jetstream.ErrKeyNotFound) {
+	if _, gerr := s.kv.Get(ctx, kvutil.AccountKey(accountID, ltID)); errors.Is(gerr, jetstream.ErrKeyNotFound) {
 		for n := range deleted {
 			if derr := s.kv.Delete(ctx, versionKey(accountID, ltID, n)); derr != nil {
 				slog.ErrorContext(ctx, "DeleteLaunchTemplateVersions: failed to remove a restored version", "launchTemplateId", ltID, "version", n, "err", derr)

spinifex/domains/ec2/launchtemplate/service_impl_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -116,7 +116,7 @@ func TestCreateLaunchTemplate_OrphanNameReclaim(t *testing.T) {
 	lt := createTemplate(t, svc, "orphan", "t3.micro")
 
 	// Simulate a crash after the name claim but with the header lost.
-	require.NoError(t, svc.kv.Delete(t.Context(), kvstore.AccountKey(testAccountID, aws.StringValue(lt.LaunchTemplateId))))
+	require.NoError(t, svc.kv.Delete(t.Context(), kvutil.AccountKey(testAccountID, aws.StringValue(lt.LaunchTemplateId))))
 
 	// The name index still points at the now-orphaned id; create must reclaim it.
 	out, err := svc.CreateLaunchTemplate(context.Background(), &ec2.CreateLaunchTemplateInput{

spinifex/domains/ec2/launchtemplate/template_lifecycle_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -38,7 +38,7 @@ func TestAccountTermination_VersionCreateRacingADeleteLeavesNoBody(t *testing.T)
 		t.Run(pause, func(t *testing.T) {
 			svc := setupTestService(t)
 			id := createWithUserData(t, svc, "web")
-			header := kvstore.AccountKey(testAccountID, id)
+			header := kvutil.AccountKey(testAccountID, id)
 			raced := false
 			race := func(key string, at string) {
 				if raced || pause != at {
@@ -71,7 +71,7 @@ func TestAccountTermination_DeleteRacingACommittedVersionRemovesIt(t *testing.T)
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvstore.AccountKey(testAccountID, id)
+	header := kvutil.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeDelete: func(key string) {
 		if key != header || raced {
@@ -95,7 +95,7 @@ func TestAccountTermination_FailedVersionCommitRemovesTheBody(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	before := accountKeys(t, svc.kv, testAccountID)
-	header := kvstore.AccountKey(testAccountID, id)
+	header := kvutil.AccountKey(testAccountID, id)
 	svc.kv = &faultyKV{KeyValue: svc.kv, failUpdate: func(key string) bool { return key == header }}
 
 	_, err := svc.CreateLaunchTemplateVersion(context.Background(), newVersionInput(id), testAccountID)
@@ -110,7 +110,7 @@ func TestAccountTermination_VersionCommitKeepsAConcurrentHeaderChange(t *testing
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvstore.AccountKey(testAccountID, id)
+	header := kvutil.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header || raced {
@@ -215,10 +215,10 @@ func TestAccountTemplateReferences_FailsClosed(t *testing.T) {
 			require.NoError(t, err)
 		},
 		"header unreadable": func(t *testing.T, svc *LaunchTemplateServiceImpl, id string) {
-			svc.kv = &faultyKV{KeyValue: svc.kv, failGet: func(key string) bool { return key == kvstore.AccountKey(testAccountID, id) }}
+			svc.kv = &faultyKV{KeyValue: svc.kv, failGet: func(key string) bool { return key == kvutil.AccountKey(testAccountID, id) }}
 		},
 		"header undecodable": func(t *testing.T, svc *LaunchTemplateServiceImpl, _ string) {
-			_, err := svc.kv.Put(t.Context(), kvstore.AccountKey(testAccountID, "lt-0broken000000000"), []byte("{not json"))
+			_, err := svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "lt-0broken000000000"), []byte("{not json"))
 			require.NoError(t, err)
 		},
 	}
@@ -241,7 +241,7 @@ func TestModifyLaunchTemplate_RacingADeleteIsNotFound(t *testing.T) {
 		t.Run(map[bool]string{false: "by-id", true: "by-name"}[byName], func(t *testing.T) {
 			svc := setupTestService(t)
 			id := createWithUserData(t, svc, "web")
-			header := kvstore.AccountKey(testAccountID, id)
+			header := kvutil.AccountKey(testAccountID, id)
 			raced := false
 			svc.kv = &faultyKV{KeyValue: svc.kv, beforeUpdate: func(key string) {
 				if key == header && !raced {
@@ -275,7 +275,7 @@ func TestModifyLaunchTemplate_RetriesAConcurrentHeaderChange(t *testing.T) {
 	_, err := svc.CreateLaunchTemplateVersion(context.Background(), newVersionInput(id), testAccountID) // v3
 	require.NoError(t, err)
 	inner := svc.kv
-	header := kvstore.AccountKey(testAccountID, id)
+	header := kvutil.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header || raced {
@@ -306,7 +306,7 @@ func TestModifyLaunchTemplate_ConflictRetriesAreBounded(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvstore.AccountKey(testAccountID, id)
+	header := kvutil.AccountKey(testAccountID, id)
 	attempts := 0
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header {
@@ -333,7 +333,7 @@ func TestModifyLaunchTemplate_ConflictReReadFailureIsInternal(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvstore.AccountKey(testAccountID, id)
+	header := kvutil.AccountKey(testAccountID, id)
 	conflicted := false
 	svc.kv = &faultyKV{
 		KeyValue: inner,
@@ -410,7 +410,7 @@ func TestDefaultChangeRacingAVersionDeleteFailsNotFound(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvstore.AccountKey(testAccountID, id)
+	header := kvutil.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header || raced {
@@ -460,7 +460,7 @@ func TestFailedVersionDeleteCommitRestoresTheBodies(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	before := accountKeys(t, svc.kv, testAccountID)
-	header := kvstore.AccountKey(testAccountID, id)
+	header := kvutil.AccountKey(testAccountID, id)
 	svc.kv = &faultyKV{KeyValue: svc.kv, failUpdate: func(key string) bool { return key == header }}
 
 	_, err := deleteVersions(svc, id, "2")
@@ -474,7 +474,7 @@ func TestFailedVersionDeleteCommitRestoresTheBodies(t *testing.T) {
 func TestVersionDeleteRacingATemplateDeleteLeavesNothing(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
-	header := kvstore.AccountKey(testAccountID, id)
+	header := kvutil.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: svc.kv, beforeUpdate: func(key string) {
 		if key != header || raced {
@@ -531,7 +531,7 @@ func TestDefaultChangeRetriesAfterAConcurrentVersionCreate(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvstore.AccountKey(testAccountID, id)
+	header := kvutil.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header || raced {

spinifex/domains/ec2/launchtemplate/template_references.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -27,7 +27,7 @@ type TemplateReferences struct {
 // header or body it cannot read or decode, is an error, because the unread
 // record may name the resource. Headerless bodies resolve from no launch.
 func (s *LaunchTemplateServiceImpl) AccountTemplateReferences(ctx context.Context, accountID string) (*TemplateReferences, error) {
-	keys, err := kvstore.Keys(ctx, s.kv)
+	keys, err := kvutil.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list launch-template keys: %w", err)
 	}

spinifex/domains/ec2/natgw/lifecycle_invariants_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	ec2eip "github.com/mulgadc/spinifex/spinifex/domains/ec2/eip"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -42,7 +42,7 @@ func TestRLC3_NatGatewayDeletesWhileRouted(t *testing.T) {
 		natgwID := createTestNatGateway(t, svc)
 
 		testutil.SeedKV(t, js, kvBucketRouteTables, map[string][]byte{
-			kvstore.AccountKey(testAccountID, "rtb-routed"): fmt.Appendf(nil,
+			kvutil.AccountKey(testAccountID, "rtb-routed"): fmt.Appendf(nil,
 				`{"route_table_id":"rtb-routed","vpc_id":"vpc-test1","routes":[{"destination_cidr_block":"0.0.0.0/0","nat_gateway_id":%q}]}`, natgwID),
 		})
 
@@ -93,7 +93,7 @@ func createTestNatGateway(t *testing.T, svc *NatGatewayServiceImpl) string {
 
 func readEIP(t *testing.T, svc *NatGatewayServiceImpl, allocID string) ec2eip.EIPRecord {
 	t.Helper()
-	entry, err := svc.eipKV.Get(t.Context(), kvstore.AccountKey(testAccountID, allocID))
+	entry, err := svc.eipKV.Get(t.Context(), kvutil.AccountKey(testAccountID, allocID))
 	require.NoError(t, err)
 	var eip ec2eip.EIPRecord
 	require.NoError(t, json.Unmarshal(entry.Value(), &eip))

spinifex/domains/ec2/natgw/service_impl.go

@@ -20,7 +20,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -45,7 +45,7 @@ func NewNatGatewayServiceImplWithNATS(ctx context.Context, natsConn *nats.Conn)
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	natgwKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketNatGateways, 10)
+	natgwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketNatGateways, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketNatGateways, err)
 	}
@@ -61,17 +61,17 @@ func NewNatGatewayServiceImplWithNATS(ctx context.Context, natsConn *nats.Conn)
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketDeletedNatGateways, err)
 	}
 
-	eipKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
+	eipKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get EIP KV bucket: %w", err)
 	}
 
-	subnetKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketSubnets, 10)
+	subnetKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketSubnets, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get subnet KV bucket: %w", err)
 	}
 
-	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
@@ -92,7 +92,7 @@ func NewNatGatewayServiceImplWithNATS(ctx context.Context, natsConn *nats.Conn)
 // for as long as a client may still ask about it.
 func getOrCreateDeletedBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) {
 	// Terraform polls DescribeNatGateways after delete and expects state=deleted.
-	return kvstore.GetOrCreateBucketWithOptions(ctx, js, kvstore.BucketOptions{
+	return kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{
 		Name:        KVBucketDeletedNatGateways,
 		Description: "Deleted NAT Gateways (auto-expire after 1 hour)",
 		History:     1,
@@ -113,7 +113,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 	allocID := *input.AllocationId
 
 	// Validate subnet exists and get its VPC
-	subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, subnetID))
+	subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, subnetID))
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound)
 	}
@@ -123,7 +123,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 	}
 
 	// Validate EIP exists and is not already associated
-	eipEntry, err := s.eipKV.Get(ctx, kvstore.AccountKey(accountID, allocID))
+	eipEntry, err := s.eipKV.Get(ctx, kvutil.AccountKey(accountID, allocID))
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidAllocationIDNotFound)
 	}
@@ -152,7 +152,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.natgwKV.Put(ctx, kvstore.AccountKey(accountID, natgwID), data); err != nil {
+	if _, err := s.natgwKV.Put(ctx, kvutil.AccountKey(accountID, natgwID), data); err != nil {
 		slog.ErrorContext(ctx, "Failed to store NAT Gateway", "natGatewayId", natgwID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
@@ -165,7 +165,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 	eipRecord.VpcId = subnetRecord.VpcId
 	eipRecord.State = "associated"
 	if eipData, err := json.Marshal(eipRecord); err == nil {
-		if _, err := s.eipKV.Update(ctx, kvstore.AccountKey(accountID, allocID), eipData, eipEntry.Revision()); err != nil {
+		if _, err := s.eipKV.Update(ctx, kvutil.AccountKey(accountID, allocID), eipData, eipEntry.Revision()); err != nil {
 			slog.WarnContext(ctx, "CreateNatGateway: failed to mark EIP associated", "natGatewayId", natgwID, "allocationId", allocID, "err", err)
 		}
 	}
@@ -185,7 +185,7 @@ func (s *NatGatewayServiceImpl) DeleteNatGateway(ctx context.Context, input *ec2
 	}
 
 	natgwID := *input.NatGatewayId
-	key := kvstore.AccountKey(accountID, natgwID)
+	key := kvutil.AccountKey(accountID, natgwID)
 
 	entry, err := s.natgwKV.Get(ctx, key)
 	if err != nil {
@@ -255,7 +255,7 @@ func (s *NatGatewayServiceImpl) publishDeleteEventsForNatGateway(ctx context.Con
 		slog.WarnContext(ctx, "DeleteNatGateway: JetStream setup failed, SNAT teardown deferred to route delete", "natGatewayId", record.NatGatewayId, "err", err)
 		return
 	}
-	rtbKV, err := kvstore.GetOrCreateBucket(ctx, js, kvBucketRouteTables, 10)
+	rtbKV, err := kvutil.GetOrCreateBucket(ctx, js, kvBucketRouteTables, 10)
 	if err != nil {
 		slog.WarnContext(ctx, "DeleteNatGateway: route-table scan failed, SNAT teardown deferred to route delete", "natGatewayId", record.NatGatewayId, "err", err)
 		return
@@ -266,7 +266,7 @@ func (s *NatGatewayServiceImpl) publishDeleteEventsForNatGateway(ctx context.Con
 	}
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := rtbKV.Get(ctx, key)
@@ -302,7 +302,7 @@ func (s *NatGatewayServiceImpl) publishDeleteEventsForNatGateway(ctx context.Con
 			if assoc.SubnetId == "" {
 				continue
 			}
-			subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, assoc.SubnetId))
+			subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, assoc.SubnetId))
 			if err != nil {
 				continue
 			}
@@ -322,7 +322,7 @@ func (s *NatGatewayServiceImpl) disassociateEIP(ctx context.Context, record *Nat
 	if record.AllocationId == "" {
 		return
 	}
-	key := kvstore.AccountKey(accountID, record.AllocationId)
+	key := kvutil.AccountKey(accountID, record.AllocationId)
 	entry, err := s.eipKV.Get(ctx, key)
 	if err != nil {
 		if !errors.Is(err, jetstream.ErrKeyNotFound) {
@@ -380,7 +380,7 @@ func (s *NatGatewayServiceImpl) DescribeNatGateways(ctx context.Context, input *
 	foundIDs := make(map[string]bool)
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 
@@ -415,7 +415,7 @@ func (s *NatGatewayServiceImpl) DescribeNatGateways(ctx context.Context, input *
 		if foundIDs[id] {
 			continue
 		}
-		key := kvstore.AccountKey(accountID, id)
+		key := kvutil.AccountKey(accountID, id)
 		entry, err := s.deletedNatgwKV.Get(ctx, key)
 		if err != nil {
 			if errors.Is(err, jetstream.ErrKeyNotFound) {

spinifex/domains/ec2/natgw/service_impl_test.go

@@ -11,7 +11,7 @@ import (
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -31,18 +31,18 @@ func setupTestServiceJS(t *testing.T) (*NatGatewayServiceImpl, jetstream.JetStre
 
 	// Seed VPC KV
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, map[string][]byte{
-		kvstore.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`),
+		kvutil.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`),
 	})
 
 	// Seed subnet KV (public subnet)
 	testutil.SeedKV(t, js, ec2vpc.KVBucketSubnets, map[string][]byte{
-		kvstore.AccountKey(testAccountID, "subnet-pub1"): []byte(`{"subnet_id":"subnet-pub1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available","map_public_ip_on_launch":true}`),
+		kvutil.AccountKey(testAccountID, "subnet-pub1"): []byte(`{"subnet_id":"subnet-pub1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available","map_public_ip_on_launch":true}`),
 	})
 
 	// Seed EIP KV
 	testutil.SeedKV(t, js, ec2eip.KVBucketEIPs, map[string][]byte{
-		kvstore.AccountKey(testAccountID, "eipalloc-test1"): []byte(`{"allocation_id":"eipalloc-test1","public_ip":"203.0.113.50","state":"allocated"}`),
-		kvstore.AccountKey(testAccountID, "eipalloc-used"):  []byte(`{"allocation_id":"eipalloc-used","public_ip":"203.0.113.51","state":"associated","association_id":"eipassoc-xxx"}`),
+		kvutil.AccountKey(testAccountID, "eipalloc-test1"): []byte(`{"allocation_id":"eipalloc-test1","public_ip":"203.0.113.50","state":"allocated"}`),
+		kvutil.AccountKey(testAccountID, "eipalloc-used"):  []byte(`{"allocation_id":"eipalloc-used","public_ip":"203.0.113.51","state":"associated","association_id":"eipassoc-xxx"}`),
 	})
 
 	svc, err := NewNatGatewayServiceImplWithNATS(t.Context(), nc)
@@ -170,7 +170,7 @@ func TestDeleteNatGateway(t *testing.T) {
 	assert.Equal(t, natgwID, *deleteOut.NatGatewayId)
 
 	// Should be gone
-	_, err = svc.natgwKV.Get(t.Context(), kvstore.AccountKey(testAccountID, natgwID))
+	_, err = svc.natgwKV.Get(t.Context(), kvutil.AccountKey(testAccountID, natgwID))
 	assert.ErrorIs(t, err, jetstream.ErrKeyNotFound)
 }
 

spinifex/domains/ec2/placementgroup/account_cleanup.go

@@ -8,7 +8,7 @@ import (
 	"log/slog"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -39,7 +39,7 @@ type AccountCleanup struct {
 // referenced returns a reason is kept; unreadable records are left for the owner scan to report. A
 // listing failure is returned and nothing is deleted.
 func (s *PlacementGroupServiceImpl) CleanupAccount(ctx context.Context, accountID string, referenced func(groupID, groupName string) string) (*AccountCleanup, error) {
-	keys, err := kvstore.Keys(ctx, s.kv)
+	keys, err := kvutil.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list placement-group keys: %w", err)
 	}
@@ -47,7 +47,7 @@ func (s *PlacementGroupServiceImpl) CleanupAccount(ctx context.Context, accountI
 	prefix := accountID + "."
 	out := &AccountCleanup{Referenced: map[string]string{}}
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.kv.Get(ctx, key)

spinifex/domains/ec2/placementgroup/account_scan.go

@@ -8,7 +8,7 @@ import (
 	"log/slog"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -34,7 +34,7 @@ type AccountScan struct {
 // A listing failure is returned as an error; per-record read failures are
 // counted. The bucket's schema-version key belongs to no account.
 func (s *PlacementGroupServiceImpl) ScanAccount(ctx context.Context, accountID string) (*AccountScan, error) {
-	keys, err := kvstore.Keys(ctx, s.kv)
+	keys, err := kvutil.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list placement-group keys: %w", err)
 	}
@@ -42,7 +42,7 @@ func (s *PlacementGroupServiceImpl) ScanAccount(ctx context.Context, accountID s
 	prefix := accountID + "."
 	scan := &AccountScan{}
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.kv.Get(ctx, key)

spinifex/domains/ec2/placementgroup/account_scan_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit/readonlystore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -42,14 +42,14 @@ func TestScanAccountClassifiesPlacementGroupRecords(t *testing.T) {
 		GroupName: "held", NodeInstances: map[string][]string{"node-2": {"i-0bbb", "i-0aaa"}, "node-3": {"i-0aaa"}},
 	}, testAccountID)
 	require.NoError(t, err)
-	_, err = svc.kv.Put(t.Context(), kvstore.AccountKey(testAccountID, "undecodable"), []byte("{not json"))
+	_, err = svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "undecodable"), []byte("{not json"))
 	require.NoError(t, err)
-	_, err = svc.kv.Put(t.Context(), kvstore.AccountKey(testAccountID, "anonymous"), []byte(`{"group_name":"anonymous"}`))
+	_, err = svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "anonymous"), []byte(`{"group_name":"anonymous"}`))
 	require.NoError(t, err)
 	createTestGroup(t, svc, "unreadable", ec2.PlacementStrategySpread)
-	require.Contains(t, bucketKeys(t, svc.kv), kvstore.VersionKey)
+	require.Contains(t, bucketKeys(t, svc.kv), kvutil.VersionKey)
 
-	svc.kv = &hookedKV{KeyValue: readonlystore.NewKV(t, svc.kv), failGet: kvstore.AccountKey(testAccountID, "unreadable")}
+	svc.kv = &hookedKV{KeyValue: readonlystore.NewKV(t, svc.kv), failGet: kvutil.AccountKey(testAccountID, "unreadable")}
 	scan, err := svc.ScanAccount(context.Background(), testAccountID)
 	require.NoError(t, err)
 

spinifex/domains/ec2/placementgroup/account_termination_current_behaviour_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -84,7 +84,7 @@ func TestCurrentBehaviour_AccountTermination_PlacementGroupStorageLayout(t *test
 	}, testAccountID)
 	require.NoError(t, err)
 
-	assert.Equal(t, []string{testAccountID + ".spread-a", kvstore.VersionKey}, bucketKeys(t, svc.kv))
+	assert.Equal(t, []string{testAccountID + ".spread-a", kvutil.VersionKey}, bucketKeys(t, svc.kv))
 
 	entry, err := svc.kv.Get(t.Context(), testAccountID+".spread-a")
 	require.NoError(t, err)
@@ -100,7 +100,7 @@ func TestCurrentBehaviour_AccountTermination_PlacementGroupStorageLayout(t *test
 // a record cannot be removed through DeletePlacementGroup either, and a read
 // failure is an internal error, never an unknown group.
 func TestCurrentBehaviour_AccountTermination_DescribePlacementGroupsHidesUnreadableRecords(t *testing.T) {
-	brokenKey := kvstore.AccountKey(testAccountID, "broken")
+	brokenKey := kvutil.AccountKey(testAccountID, "broken")
 	tests := []struct {
 		name       string
 		seed       func(t *testing.T, svc *PlacementGroupServiceImpl)
@@ -139,7 +139,7 @@ func TestCurrentBehaviour_AccountTermination_DescribePlacementGroupsHidesUnreada
 func TestAccountTermination_DeletePlacementGroupLosesToAConcurrentMember(t *testing.T) {
 	svc := setupTestService(t)
 	createTestGroup(t, svc, "cluster-a", ec2.PlacementStrategyCluster)
-	key := kvstore.AccountKey(testAccountID, "cluster-a")
+	key := kvutil.AccountKey(testAccountID, "cluster-a")
 
 	launcher := &PlacementGroupServiceImpl{kv: svc.kv, region: svc.region}
 	var reserveErr, finalizeErr error
@@ -222,7 +222,7 @@ func TestAccountCleanup_DeletesEmptyGroupsAndBlocksOnMembers(t *testing.T) {
 	empty := createTestGroup(t, svc, "empty", ec2.PlacementStrategySpread)
 	held := createTestGroup(t, svc, "held", ec2.PlacementStrategyCluster)
 	require.NoError(t, addMember(t, svc, "held", "i-0123456789abcdef0"))
-	_, err := svc.kv.Put(t.Context(), kvstore.AccountKey(testAccountID, "broken"), []byte("{not json"))
+	_, err := svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "broken"), []byte("{not json"))
 	require.NoError(t, err)
 	other := "999988887777"
 	_, err = svc.CreatePlacementGroup(context.Background(), &ec2.CreatePlacementGroupInput{
@@ -238,7 +238,7 @@ func TestAccountCleanup_DeletesEmptyGroupsAndBlocksOnMembers(t *testing.T) {
 
 	assert.Equal(t, []string{"held"}, describedGroupNames(t, svc, testAccountID))
 	assert.Equal(t, []string{"theirs"}, describedGroupNames(t, svc, other))
-	_, err = svc.kv.Get(t.Context(), kvstore.AccountKey(testAccountID, "broken"))
+	_, err = svc.kv.Get(t.Context(), kvutil.AccountKey(testAccountID, "broken"))
 	require.NoError(t, err, "an unreadable record is never deleted")
 }
 
@@ -247,7 +247,7 @@ func TestAccountCleanup_DeletesEmptyGroupsAndBlocksOnMembers(t *testing.T) {
 func TestAccountCleanup_ConcurrentMemberBlocksTheDelete(t *testing.T) {
 	svc := setupTestService(t)
 	group := createTestGroup(t, svc, "cluster-a", ec2.PlacementStrategyCluster)
-	key := kvstore.AccountKey(testAccountID, "cluster-a")
+	key := kvutil.AccountKey(testAccountID, "cluster-a")
 	launcher := &PlacementGroupServiceImpl{kv: svc.kv, region: svc.region}
 	var memberErr error
 	// Read 1 is the cleanup's listing read; read 2 is the delete's check.

spinifex/domains/ec2/placementgroup/service_impl.go

@@ -17,7 +17,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -87,7 +87,7 @@ func NewPlacementGroupServiceImplWithNATS(ctx context.Context, cfg *config.Confi
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketPlacementGroups, 10)
+	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketPlacementGroups, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketPlacementGroups, err)
 	}
@@ -137,7 +137,7 @@ func (s *PlacementGroupServiceImpl) CreatePlacementGroup(ctx context.Context, in
 	}
 
 	groupName := *input.GroupName
-	key := kvstore.AccountKey(accountID, groupName)
+	key := kvutil.AccountKey(accountID, groupName)
 	groupID := awsidentifiers.GenerateResourceID("pg")
 
 	record := PlacementGroupRecord{
@@ -175,7 +175,7 @@ func (s *PlacementGroupServiceImpl) DeletePlacementGroup(ctx context.Context, in
 	}
 
 	groupName := *input.GroupName
-	if _, err := s.deleteGroup(ctx, accountID, kvstore.AccountKey(accountID, groupName)); err != nil {
+	if _, err := s.deleteGroup(ctx, accountID, kvutil.AccountKey(accountID, groupName)); err != nil {
 		if errors.Is(err, errPGAbsent) {
 			// Lower case, unlike DescribePlacementGroups' message for the same code.
 			return nil, awserrors.Errorf(awserrors.ErrorInvalidPlacementGroupUnknown, "The placement group '%s' is unknown.", groupName)
@@ -194,7 +194,7 @@ func (s *PlacementGroupServiceImpl) DeletePlacementGroup(ctx context.Context, in
 // The returned count is the members seen by the last check.
 func (s *PlacementGroupServiceImpl) deleteGroup(ctx context.Context, accountID, key string) (int, error) {
 	members := 0
-	record, err := kvstore.DeleteIf(ctx, s.kv, key, casConfig(), func(record *PlacementGroupRecord) error {
+	record, err := kvutil.DeleteIf(ctx, s.kv, key, casConfig(), func(record *PlacementGroupRecord) error {
 		members = memberCount(record.NodeInstances)
 		if members > 0 {
 			return errPGInUse
@@ -238,7 +238,7 @@ func (s *PlacementGroupServiceImpl) DescribePlacementGroups(ctx context.Context,
 
 	var groups []*ec2.PlacementGroup
 	for _, k := range keys {
-		if k == kvstore.VersionKey {
+		if k == kvutil.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -341,7 +341,7 @@ func pgMatchesFilters(record *PlacementGroupRecord, filters map[string][]string)
 
 const maxCASRetries = 5
 
-// errPGAbsent and errPGContended mark the two kvstore outcomes that map to
+// errPGAbsent and errPGContended mark the two kvutil outcomes that map to
 // specific AWS errors rather than a generic internal failure, and neither
 // escapes this file. errPGInUse is the AWS error itself.
 var (
@@ -350,8 +350,8 @@ var (
 	errPGInUse     = errors.New(awserrors.ErrorInvalidPlacementGroupInUse)
 )
 
-// mapCASError turns a kvstore failure into the AWS error the caller returns.
-// A non-kvstore error came from mutate and is already an AWS code.
+// mapCASError turns a kvutil failure into the AWS error the caller returns.
+// A non-kvutil error came from mutate and is already an AWS code.
 func mapCASError(ctx context.Context, op, groupName, accountID string, err error) error {
 	switch {
 	case errors.Is(err, errPGAbsent):
@@ -359,16 +359,16 @@ func mapCASError(ctx context.Context, op, groupName, accountID string, err error
 	case errors.Is(err, errPGContended):
 		slog.ErrorContext(ctx, "Placement group CAS retries exhausted under contention",
 			"op", op, "groupName", groupName, "accountID", accountID)
-	case errors.Is(err, kvstore.ErrRead):
+	case errors.Is(err, kvutil.ErrRead):
 		slog.ErrorContext(ctx, "Failed to read placement group from KV",
 			"op", op, "groupName", groupName, "accountID", accountID, "err", err)
-	case errors.Is(err, kvstore.ErrDecode):
+	case errors.Is(err, kvutil.ErrDecode):
 		slog.ErrorContext(ctx, "Corrupt placement group record in KV",
 			"op", op, "groupName", groupName, "accountID", accountID, "err", err)
-	case errors.Is(err, kvstore.ErrEncode):
+	case errors.Is(err, kvutil.ErrEncode):
 		slog.ErrorContext(ctx, "Failed to marshal placement group record",
 			"op", op, "groupName", groupName, "accountID", accountID, "err", err)
-	case errors.Is(err, kvstore.ErrWrite):
+	case errors.Is(err, kvutil.ErrWrite):
 		slog.ErrorContext(ctx, "Failed to write placement group to KV",
 			"op", op, "groupName", groupName, "accountID", accountID, "err", err)
 	default:
@@ -378,8 +378,8 @@ func mapCASError(ctx context.Context, op, groupName, accountID string, err error
 }
 
 // casConfig is the shared CAS policy for every placement group mutation.
-func casConfig() kvstore.CASConfig {
-	return kvstore.CASConfig{
+func casConfig() kvutil.CASConfig {
+	return kvutil.CASConfig{
 		Attempts:  maxCASRetries,
 		NotFound:  errPGAbsent,
 		Exhausted: func(string, int) error { return errPGContended },
@@ -394,7 +394,7 @@ func (s *PlacementGroupServiceImpl) ReserveSpreadNodes(ctx context.Context, inpu
 	}
 
 	var selected []string
-	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			if record.State != ec2.PlacementGroupStateAvailable {
 				return false, errors.New(awserrors.ErrorInvalidPlacementGroupUnknown)
@@ -437,7 +437,7 @@ func (s *PlacementGroupServiceImpl) FinalizeSpreadInstances(ctx context.Context,
 		return nil, errors.New(awserrors.ErrorMissingParameter)
 	}
 
-	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			maps.Copy(record.NodeInstances, input.NodeInstances)
 			return true, nil
@@ -462,7 +462,7 @@ func (s *PlacementGroupServiceImpl) ReleaseSpreadNodes(ctx context.Context, inpu
 		releaseSet[n] = true
 	}
 
-	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			changed := false
 			for node := range releaseSet {
@@ -490,7 +490,7 @@ func (s *PlacementGroupServiceImpl) RemoveInstance(ctx context.Context, input *R
 		return nil, errors.New(awserrors.ErrorMissingParameter)
 	}
 
-	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			instances, exists := record.NodeInstances[input.NodeName]
 			if !exists {
@@ -533,7 +533,7 @@ func (s *PlacementGroupServiceImpl) ReserveClusterNode(ctx context.Context, inpu
 	}
 
 	var targetNode string
-	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			if record.State != ec2.PlacementGroupStateAvailable {
 				return false, errors.New(awserrors.ErrorInvalidPlacementGroupUnknown)
@@ -570,7 +570,7 @@ func (s *PlacementGroupServiceImpl) FinalizeClusterInstances(ctx context.Context
 		return nil, errors.New(awserrors.ErrorMissingParameter)
 	}
 
-	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			// Append rather than replace: a cluster group can have several launches in flight at once.
 			for node, ids := range input.NodeInstances {

spinifex/domains/ec2/placementgroup/service_impl_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -39,7 +39,7 @@ func createTestGroup(t *testing.T, svc *PlacementGroupServiceImpl, name, strateg
 // readRecord returns the stored record for assertions.
 func readRecord(t *testing.T, svc *PlacementGroupServiceImpl, groupName string) *PlacementGroupRecord {
 	t.Helper()
-	entry, err := svc.kv.Get(t.Context(), kvstore.AccountKey(testAccountID, groupName))
+	entry, err := svc.kv.Get(t.Context(), kvutil.AccountKey(testAccountID, groupName))
 	require.NoError(t, err)
 	var record PlacementGroupRecord
 	require.NoError(t, json.Unmarshal(entry.Value(), &record))
@@ -50,7 +50,7 @@ func readRecord(t *testing.T, svc *PlacementGroupServiceImpl, groupName string)
 // starting state the public API cannot produce.
 func seedRecord(t *testing.T, svc *PlacementGroupServiceImpl, groupName string, mutate func(*PlacementGroupRecord)) {
 	t.Helper()
-	_, err := kvstore.Update(t.Context(), svc.kv, kvstore.AccountKey(testAccountID, groupName), kvstore.CASConfig{},
+	_, err := kvutil.Update(t.Context(), svc.kv, kvutil.AccountKey(testAccountID, groupName), kvutil.CASConfig{},
 		func(record *PlacementGroupRecord) (bool, error) {
 			mutate(record)
 			return true, nil

spinifex/domains/ec2/routetable/service_impl.go

@@ -25,7 +25,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -52,7 +52,7 @@ func NewRouteTableServiceImplWithNATS(ctx context.Context, cfg *config.Config, n
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	rtbKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketRouteTables, 10)
+	rtbKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketRouteTables, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketRouteTables, err)
 	}
@@ -60,22 +60,22 @@ func NewRouteTableServiceImplWithNATS(ctx context.Context, cfg *config.Config, n
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketRouteTables, err)
 	}
 
-	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
 
-	igwKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2igw.KVBucketIGW, 10)
+	igwKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2igw.KVBucketIGW, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get IGW KV bucket: %w", err)
 	}
 
-	subnetKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketSubnets, 10)
+	subnetKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketSubnets, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get subnet KV bucket: %w", err)
 	}
 
-	natgwKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2natgw.KVBucketNatGateways, 10)
+	natgwKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2natgw.KVBucketNatGateways, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get NAT Gateway KV bucket: %w", err)
 	}
@@ -95,7 +95,7 @@ func NewRouteTableServiceImplWithNATS(ctx context.Context, cfg *config.Config, n
 
 // getRouteTable retrieves a route table record from KV.
 func (s *RouteTableServiceImpl) getRouteTable(ctx context.Context, accountID, rtbID string) (*RouteTableRecord, error) {
-	entry, err := s.rtbKV.Get(ctx, kvstore.AccountKey(accountID, rtbID))
+	entry, err := s.rtbKV.Get(ctx, kvutil.AccountKey(accountID, rtbID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
 			return nil, rtbNotFoundError(rtbID)
@@ -118,7 +118,7 @@ func (s *RouteTableServiceImpl) putRouteTable(ctx context.Context, accountID str
 		slog.ErrorContext(ctx, "Failed to marshal route table record", "routeTableId", record.RouteTableId, "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.rtbKV.Put(ctx, kvstore.AccountKey(accountID, record.RouteTableId), data); err != nil {
+	if _, err := s.rtbKV.Put(ctx, kvutil.AccountKey(accountID, record.RouteTableId), data); err != nil {
 		slog.ErrorContext(ctx, "Failed to write route table to KV", "routeTableId", record.RouteTableId, "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
@@ -130,7 +130,7 @@ func (s *RouteTableServiceImpl) putRouteTable(ctx context.Context, accountID str
 // parallel AssociateRouteTable calls Terraform fans out per route table.
 const rtbCASMaxRetries = 16
 
-// errRTBAbsent and errRTBContended mark the two kvstore outcomes that map to
+// errRTBAbsent and errRTBContended mark the two kvutil outcomes that map to
 // specific AWS errors rather than a generic internal failure. Neither
 // escapes this file.
 var (
@@ -143,7 +143,7 @@ var (
 // several subnets with one route table at once. mutate reports whether
 // it changed the record; a false return commits nothing.
 func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, accountID, rtbID string, mutate func(*RouteTableRecord) (bool, error)) error {
-	_, err := kvstore.Update(ctx, s.rtbKV, kvstore.AccountKey(accountID, rtbID), kvstore.CASConfig{
+	_, err := kvutil.Update(ctx, s.rtbKV, kvutil.AccountKey(accountID, rtbID), kvutil.CASConfig{
 		Attempts:  rtbCASMaxRetries,
 		NotFound:  errRTBAbsent,
 		Exhausted: func(string, int) error { return errRTBContended },
@@ -157,13 +157,13 @@ func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, account
 		// Contended rather than broken.
 		slog.ErrorContext(ctx, "Route table CAS retries exhausted under contention",
 			"routeTableId", rtbID, "attempts", rtbCASMaxRetries)
-	case errors.Is(err, kvstore.ErrRead):
+	case errors.Is(err, kvutil.ErrRead):
 		slog.ErrorContext(ctx, "Failed to read route table from KV", "routeTableId", rtbID, "err", err)
-	case errors.Is(err, kvstore.ErrDecode):
+	case errors.Is(err, kvutil.ErrDecode):
 		slog.ErrorContext(ctx, "Corrupt route table record in KV", "routeTableId", rtbID, "err", err)
-	case errors.Is(err, kvstore.ErrEncode):
+	case errors.Is(err, kvutil.ErrEncode):
 		slog.ErrorContext(ctx, "Failed to marshal route table record", "routeTableId", rtbID, "err", err)
-	case errors.Is(err, kvstore.ErrWrite):
+	case errors.Is(err, kvutil.ErrWrite):
 		slog.ErrorContext(ctx, "Failed to write route table to KV", "routeTableId", rtbID, "err", err)
 	default:
 		return err
@@ -173,7 +173,7 @@ func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, account
 
 // getVPCCidr looks up a VPC's CIDR block from the VPC KV bucket.
 func (s *RouteTableServiceImpl) getVPCCidr(ctx context.Context, accountID, vpcID string) (string, error) {
-	entry, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID))
+	entry, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID))
 	if err != nil {
 		return "", awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
 	}
@@ -229,7 +229,7 @@ func (s *RouteTableServiceImpl) subnetsImplicitlyOnMainRT(ctx context.Context, a
 	}
 	var implicit []string
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.subnetKV.Get(ctx, key)
@@ -387,7 +387,7 @@ func (s *RouteTableServiceImpl) allSubnetsForVPC(ctx context.Context, accountID,
 	}
 	var subnets []string
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.subnetKV.Get(ctx, key)
@@ -455,7 +455,7 @@ func (s *RouteTableServiceImpl) allRouteTablesForVPC(ctx context.Context, accoun
 
 	var results []RouteTableRecord
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.rtbKV.Get(ctx, key)
@@ -577,7 +577,7 @@ func (s *RouteTableServiceImpl) DeleteRouteTable(ctx context.Context, input *ec2
 		}
 	}
 
-	if err := s.rtbKV.Delete(ctx, kvstore.AccountKey(accountID, rtbID)); err != nil {
+	if err := s.rtbKV.Delete(ctx, kvutil.AccountKey(accountID, rtbID)); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -633,7 +633,7 @@ func (s *RouteTableServiceImpl) DescribeRouteTables(ctx context.Context, input *
 	foundIDs := make(map[string]bool)
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 
@@ -740,7 +740,7 @@ func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.Crea
 	case input.GatewayId != nil && *input.GatewayId != "":
 		igwID := *input.GatewayId
 		// Verify IGW exists and is attached to the same VPC
-		igwEntry, err := s.igwKV.Get(ctx, kvstore.AccountKey(accountID, igwID))
+		igwEntry, err := s.igwKV.Get(ctx, kvutil.AccountKey(accountID, igwID))
 		if err != nil {
 			return nil, gatewayNotFoundError(igwID)
 		}
@@ -770,7 +770,7 @@ func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.Crea
 	case input.NatGatewayId != nil && *input.NatGatewayId != "":
 		natgwID := *input.NatGatewayId
 		// Verify NAT GW exists and belongs to the same VPC
-		natgwEntry, err := s.natgwKV.Get(ctx, kvstore.AccountKey(accountID, natgwID))
+		natgwEntry, err := s.natgwKV.Get(ctx, kvutil.AccountKey(accountID, natgwID))
 		if err != nil {
 			return nil, errors.New(awserrors.ErrorInvalidNatGatewayIDNotFound)
 		}
@@ -924,7 +924,7 @@ func (s *RouteTableServiceImpl) ReplaceRoute(ctx context.Context, input *ec2.Rep
 	igwID := *input.GatewayId
 
 	// AWS reports a missing gateway ahead of a missing route.
-	igwEntry, err := s.igwKV.Get(ctx, kvstore.AccountKey(accountID, igwID))
+	igwEntry, err := s.igwKV.Get(ctx, kvutil.AccountKey(accountID, igwID))
 	if err != nil {
 		return nil, gatewayNotFoundError(igwID)
 	}
@@ -983,7 +983,7 @@ func (s *RouteTableServiceImpl) AssociateRouteTable(ctx context.Context, input *
 	}
 
 	// Verify subnet exists and belongs to the same VPC
-	subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, subnetID))
+	subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, subnetID))
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound)
 	}
@@ -1077,7 +1077,7 @@ func (s *RouteTableServiceImpl) DisassociateRouteTable(ctx context.Context, inpu
 	}
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 
@@ -1168,7 +1168,7 @@ func (s *RouteTableServiceImpl) ReplaceRouteTableAssociation(ctx context.Context
 	}
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 
@@ -1369,7 +1369,7 @@ func (s *RouteTableServiceImpl) publishNatGatewayDeleteEvents(ctx context.Contex
 	if s.natsConn == nil {
 		return
 	}
-	natgwEntry, err := s.natgwKV.Get(ctx, kvstore.AccountKey(accountID, natgwID))
+	natgwEntry, err := s.natgwKV.Get(ctx, kvutil.AccountKey(accountID, natgwID))
 	if err != nil {
 		slog.WarnContext(ctx, "NAT GW event: natgw lookup failed", "topic", networkv1.NATGatewayDeleteSubject, "natGatewayId", natgwID, "err", err)
 		return
@@ -1417,7 +1417,7 @@ func (s *RouteTableServiceImpl) publishNatGatewayEventsForAssociation(ctx contex
 		if r.NatGatewayId == "" {
 			continue
 		}
-		natgwEntry, err := s.natgwKV.Get(ctx, kvstore.AccountKey(accountID, r.NatGatewayId))
+		natgwEntry, err := s.natgwKV.Get(ctx, kvutil.AccountKey(accountID, r.NatGatewayId))
 		if err != nil {
 			slog.WarnContext(ctx, "NAT GW event: natgw lookup failed", "topic", topic, "natGatewayId", r.NatGatewayId, "err", err)
 			continue
@@ -1442,7 +1442,7 @@ func (s *RouteTableServiceImpl) publishNatGatewayEventForSubnet(ctx context.Cont
 	if s.natsConn == nil {
 		return
 	}
-	subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, subnetID))
+	subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, subnetID))
 	if err != nil {
 		slog.WarnContext(ctx, "NAT GW event: subnet lookup failed", "topic", topic, "subnetId", subnetID, "err", err)
 		return

spinifex/domains/ec2/routetable/service_impl_test.go

@@ -13,7 +13,7 @@ import (
 	ec2natgw "github.com/mulgadc/spinifex/spinifex/domains/ec2/natgw"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -28,23 +28,23 @@ func setupTestService(t *testing.T) *RouteTableServiceImpl {
 
 	// Seed VPC KV
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, map[string][]byte{
-		kvstore.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`),
+		kvutil.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`),
 	})
 
 	// Seed IGW KV (attached to vpc-test1)
 	testutil.SeedKV(t, js, ec2igw.KVBucketIGW, map[string][]byte{
-		kvstore.AccountKey(testAccountID, "igw-test1"): []byte(`{"internet_gateway_id":"igw-test1","vpc_id":"vpc-test1","state":"available"}`),
+		kvutil.AccountKey(testAccountID, "igw-test1"): []byte(`{"internet_gateway_id":"igw-test1","vpc_id":"vpc-test1","state":"available"}`),
 	})
 
 	// Seed subnet KV
 	testutil.SeedKV(t, js, ec2vpc.KVBucketSubnets, map[string][]byte{
-		kvstore.AccountKey(testAccountID, "subnet-test1"): []byte(`{"subnet_id":"subnet-test1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available"}`),
-		kvstore.AccountKey(testAccountID, "subnet-priv1"): []byte(`{"subnet_id":"subnet-priv1","vpc_id":"vpc-test1","cidr_block":"10.0.11.0/24","state":"available"}`),
+		kvutil.AccountKey(testAccountID, "subnet-test1"): []byte(`{"subnet_id":"subnet-test1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available"}`),
+		kvutil.AccountKey(testAccountID, "subnet-priv1"): []byte(`{"subnet_id":"subnet-priv1","vpc_id":"vpc-test1","cidr_block":"10.0.11.0/24","state":"available"}`),
 	})
 
 	// Seed NAT Gateway KV
 	testutil.SeedKV(t, js, ec2natgw.KVBucketNatGateways, map[string][]byte{
-		kvstore.AccountKey(testAccountID, "nat-test1"): []byte(`{"nat_gateway_id":"nat-test1","vpc_id":"vpc-test1","subnet_id":"subnet-test1","public_ip":"192.168.1.100","state":"available"}`),
+		kvutil.AccountKey(testAccountID, "nat-test1"): []byte(`{"nat_gateway_id":"nat-test1","vpc_id":"vpc-test1","subnet_id":"subnet-test1","public_ip":"192.168.1.100","state":"available"}`),
 	})
 
 	svc, err := NewRouteTableServiceImplWithNATS(t.Context(), nil, nc)

spinifex/domains/ec2/spotinstance/account_scan.go

@@ -10,7 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/nats-io/nats.go/jetstream"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 )
 
 // AccountSpotRequest is one persisted request in an account scan.
@@ -54,7 +54,7 @@ func scanBucket(ctx context.Context, kv jetstream.KeyValue, accountID string) ([
 	prefix := accountID + "."
 	var out []AccountSpotRequest
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := kv.Get(ctx, key)

spinifex/domains/ec2/spotinstance/account_scan_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 )
 
 func TestScanAccountListsActiveAndTerminalForOneAccount(t *testing.T) {
@@ -35,7 +35,7 @@ func TestScanAccountListsActiveAndTerminalForOneAccount(t *testing.T) {
 func TestScanAccountFailsOnAnUndecodableRecord(t *testing.T) {
 	svc := setupTestService(t)
 	putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"))
-	_, err := svc.activeKV.Put(t.Context(), kvstore.AccountKey(testAccountID, "sir-bad"), []byte("{not json"))
+	_, err := svc.activeKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-bad"), []byte("{not json"))
 	require.NoError(t, err)
 
 	scan, err := svc.ScanAccount(t.Context(), testAccountID)
@@ -45,7 +45,7 @@ func TestScanAccountFailsOnAnUndecodableRecord(t *testing.T) {
 
 func TestScanAccountFailsOnARecordWithoutARequest(t *testing.T) {
 	svc := setupTestService(t)
-	_, err := svc.terminalKV.Put(t.Context(), kvstore.AccountKey(testAccountID, "sir-empty"), []byte(`{"account_id":"x"}`))
+	_, err := svc.terminalKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-empty"), []byte(`{"account_id":"x"}`))
 	require.NoError(t, err)
 
 	_, err = svc.ScanAccount(t.Context(), testAccountID)
@@ -76,7 +76,7 @@ func (k failingGetKV) Get(ctx context.Context, key string) (jetstream.KeyValueEn
 func TestScanAccountFailsWhenARecordCannotBeRead(t *testing.T) {
 	svc := setupTestService(t)
 	putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"), makeSIR("sir-bbb", "i-bbb"))
-	svc.activeKV = failingGetKV{KeyValue: svc.activeKV, key: kvstore.AccountKey(testAccountID, "sir-bbb")}
+	svc.activeKV = failingGetKV{KeyValue: svc.activeKV, key: kvutil.AccountKey(testAccountID, "sir-bbb")}
 
 	scan, err := svc.ScanAccount(t.Context(), testAccountID)
 	require.Error(t, err)

spinifex/domains/ec2/spotinstance/launch_references.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -24,14 +24,14 @@ type LaunchReferences struct {
 // account. It is complete or an error: an unlistable bucket or an unreadable
 // request is an error, because the unread request may name the resource.
 func (s *SpotInstanceServiceImpl) AccountLaunchReferences(ctx context.Context, accountID string) (*LaunchReferences, error) {
-	keys, err := kvstore.Keys(ctx, s.activeKV)
+	keys, err := kvutil.Keys(ctx, s.activeKV)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list spot requests: %w", err)
 	}
 	prefix := accountID + "."
 	out := &LaunchReferences{}
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		record, err := s.readRecord(ctx, s.activeKV, key)

spinifex/domains/ec2/spotinstance/launch_references_test.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -41,7 +41,7 @@ func TestAccountLaunchReferences_ReadsOpenAndActiveRequests(t *testing.T) {
 func TestAccountLaunchReferences_UnreadableRequestFailsTheCheck(t *testing.T) {
 	svc := setupTestService(t)
 	putSIRs(t, svc, makeSIR("sir-active01", "i-0000000000000001"))
-	_, err := svc.activeKV.Put(t.Context(), kvstore.AccountKey(testAccountID, "sir-broken01"), []byte("{not json"))
+	_, err := svc.activeKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-broken01"), []byte("{not json"))
 	require.NoError(t, err)
 
 	_, err = svc.AccountLaunchReferences(t.Context(), testAccountID)

spinifex/domains/ec2/spotinstance/service_impl.go

@@ -15,7 +15,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -41,7 +41,7 @@ func NewSpotInstanceServiceImplWithNATS(ctx context.Context, cfg *config.Config,
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	activeKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketSpotRequests, 10)
+	activeKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketSpotRequests, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSpotRequests, err)
 	}
@@ -71,7 +71,7 @@ func NewSpotInstanceServiceImplWithNATS(ctx context.Context, cfg *config.Config,
 // getOrCreateTerminalBucket opens the terminal bucket without changing its
 // existing replica or placement configuration, creating it only when absent.
 func getOrCreateTerminalBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) {
-	return kvstore.GetOrCreateBucketWithOptions(ctx, js, kvstore.BucketOptions{
+	return kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{
 		Name:        KVBucketSpotRequestsTerminal,
 		Description: "Terminal Spot Instance Requests (auto-expire after 1 hour)",
 		History:     1,
@@ -91,7 +91,7 @@ func (s *SpotInstanceServiceImpl) PutSpotInstanceRequests(ctx context.Context, i
 		if err != nil {
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
-		if _, err := s.activeKV.Put(ctx, kvstore.AccountKey(accountID, sirID), data); err != nil {
+		if _, err := s.activeKV.Put(ctx, kvutil.AccountKey(accountID, sirID), data); err != nil {
 			slog.ErrorContext(ctx, "PutSpotInstanceRequests: KV put failed", "sirId", sirID, "err", err)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
@@ -171,7 +171,7 @@ func (s *SpotInstanceServiceImpl) CancelSpotInstanceRequests(ctx context.Context
 	cancelled := make([]*ec2.CancelledSpotInstanceRequest, 0, len(input.SpotInstanceRequestIds))
 	for _, idPtr := range input.SpotInstanceRequestIds {
 		sirID := aws.StringValue(idPtr)
-		key := kvstore.AccountKey(accountID, sirID)
+		key := kvutil.AccountKey(accountID, sirID)
 
 		record, err := s.readRecord(ctx, s.activeKV, key)
 		if err == nil {
@@ -213,7 +213,7 @@ func (s *SpotInstanceServiceImpl) CloseForInstance(ctx context.Context, instance
 
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		record, err := s.readRecord(ctx, s.activeKV, key)
@@ -251,7 +251,7 @@ func (s *SpotInstanceServiceImpl) listRequests(ctx context.Context, kv jetstream
 	prefix := accountID + "."
 	var requests []*ec2.SpotInstanceRequest
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		record, err := s.readRecord(ctx, kv, key)

spinifex/domains/ec2/tagmirror/tagging_current_behaviour_test.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -39,7 +39,7 @@ func (k *interleavingKV) Get(ctx context.Context, key string) (jetstream.KeyValu
 // its write both survive alongside its own tag.
 func TestCurrentBehaviour_Tagging_RecordMirrorKeepsInterleavedWrites(t *testing.T) {
 	const acct, id = "123456789012", "sg-0123456789abcdef0"
-	key := kvstore.AccountKey(acct, id)
+	key := kvutil.AccountKey(acct, id)
 	seed, err := json.Marshal(tagMirrorRecord{Name: "original", Tags: map[string]string{}})
 	require.NoError(t, err)
 	kv := seedKV(t, "tagmirror-cb", map[string][]byte{key: seed})

spinifex/domains/ec2/tagmirror/tagmirror.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -83,8 +83,8 @@ var errRecordAbsent = errors.New("tagmirror: record absent")
 // applying mut and committing at the read revision, retrying on conflict so a
 // concurrent write is never reverted. An absent resource is skipped.
 func UpdateKVRecordTags[R any](ctx context.Context, kv jetstream.KeyValue, accountID, resourceID string, mut func(*R)) error {
-	key := kvstore.AccountKey(accountID, resourceID)
-	_, err := kvstore.Update(ctx, kv, key, kvstore.CASConfig{NotFound: errRecordAbsent}, func(rec *R) (bool, error) {
+	key := kvutil.AccountKey(accountID, resourceID)
+	_, err := kvutil.Update(ctx, kv, key, kvutil.CASConfig{NotFound: errRecordAbsent}, func(rec *R) (bool, error) {
 		mut(rec)
 		return true, nil
 	})

spinifex/domains/ec2/tagmirror/tagmirror_test.go

@@ -8,7 +8,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -75,15 +75,15 @@ func TestUpdateKVRecordTags(t *testing.T) {
 	rec, err := json.Marshal(&tagMirrorRecord{Name: "r1", Tags: map[string]string{"a": "1"}})
 	require.NoError(t, err)
 	kv := seedKV(t, "tagmirror-test", map[string][]byte{
-		kvstore.AccountKey("acct", "res-1"):   rec,
-		kvstore.AccountKey("acct", "res-bad"): []byte("not-json"),
+		kvutil.AccountKey("acct", "res-1"):   rec,
+		kvutil.AccountKey("acct", "res-bad"): []byte("not-json"),
 	})
 
 	t.Run("mutates and persists", func(t *testing.T) {
 		require.NoError(t, UpdateKVRecordTags(t.Context(), kv, "acct", "res-1", func(r *tagMirrorRecord) {
 			r.Tags["b"] = "2"
 		}))
-		entry, err := kv.Get(t.Context(), kvstore.AccountKey("acct", "res-1"))
+		entry, err := kv.Get(t.Context(), kvutil.AccountKey("acct", "res-1"))
 		require.NoError(t, err)
 		var got tagMirrorRecord
 		require.NoError(t, json.Unmarshal(entry.Value(), &got))
@@ -110,14 +110,14 @@ func TestMirrorKVRecordTags(t *testing.T) {
 	nilTags, err := json.Marshal(&tagMirrorRecord{})
 	require.NoError(t, err)
 	kv := seedKV(t, "tagmirror-mirror-test", map[string][]byte{
-		kvstore.AccountKey("acct", "vol-1"): withTags,
-		kvstore.AccountKey("acct", "vol-2"): nilTags,
+		kvutil.AccountKey("acct", "vol-1"): withTags,
+		kvutil.AccountKey("acct", "vol-2"): nilTags,
 	})
 
 	tagsOf := func(r *tagMirrorRecord) *map[string]string { return &r.Tags }
 
 	readTags := func(t *testing.T, id string) map[string]string {
-		entry, err := kv.Get(t.Context(), kvstore.AccountKey("acct", id))
+		entry, err := kv.Get(t.Context(), kvutil.AccountKey("acct", id))
 		require.NoError(t, err)
 		var got tagMirrorRecord
 		require.NoError(t, json.Unmarshal(entry.Value(), &got))
@@ -150,7 +150,7 @@ func TestMirrorKVRecordTags(t *testing.T) {
 	// must never be touched even though it exists in this bucket.
 	otherRec, err := json.Marshal(&tagMirrorRecord{Tags: map[string]string{"x": "y"}})
 	require.NoError(t, err)
-	_, err = kv.Put(t.Context(), kvstore.AccountKey("acct", "snap-other"), otherRec)
+	_, err = kv.Put(t.Context(), kvutil.AccountKey("acct", "snap-other"), otherRec)
 	require.NoError(t, err)
 	require.NoError(t, MirrorKVRecordTags(t.Context(), kv, "acct", "vol-", resources, tagsOf,
 		MergeTagsMut(&ec2.CreateTagsInput{Tags: []*ec2.Tag{

spinifex/domains/ec2/tags/service_impl.go

@@ -23,7 +23,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -274,7 +274,7 @@ func (s *TagsServiceImpl) CreateTags(ctx context.Context, input *ec2.CreateTagsI
 func (s *TagsServiceImpl) taggedResourceIDs(ctx context.Context, accountID string) ([]string, error) {
 	seen := make(map[string]struct{})
 
-	keys, err := kvstore.Keys(ctx, s.kv)
+	keys, err := kvutil.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list tag keys: %w", err)
 	}

spinifex/domains/ec2/tags/store.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -46,7 +46,7 @@ func resourceIDFromKVKey(key, accountID string) (string, bool) {
 // GetOrCreateTagsBucket opens the tag bucket, creating it at the cluster's
 // replica count if this is the first node to reach it.
 func GetOrCreateTagsBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) {
-	return kvstore.GetOrCreateBucket(ctx, js, KVBucketTags, kvTagsHistory)
+	return kvutil.GetOrCreateBucket(ctx, js, KVBucketTags, kvTagsHistory)
 }
 
 // readKVTags returns a resource's tags from the bucket, and whether the key
@@ -76,7 +76,7 @@ func (s *TagsServiceImpl) readKVTags(ctx context.Context, accountID, resourceID
 // before the store moved.
 func (s *TagsServiceImpl) mutateTags(ctx context.Context, accountID, resourceID string, mutate func(map[string]string)) error {
 	key := tagsKVKey(accountID, resourceID)
-	_, err := kvstore.Update(ctx, s.kv, key, kvstore.CASConfig{
+	_, err := kvutil.Update(ctx, s.kv, key, kvutil.CASConfig{
 		Attempts:       tagCASAttempts,
 		CreateIfAbsent: true,
 		Exhausted: func(key string, attempts int) error {
@@ -104,7 +104,7 @@ func (s *TagsServiceImpl) PutResourceTags(ctx context.Context, accountID, resour
 	if err != nil {
 		return err
 	}
-	return kvstore.Put(ctx, s.kv, tagsKVKey(accountID, resourceID), kvstore.CASConfig{
+	return kvutil.Put(ctx, s.kv, tagsKVKey(accountID, resourceID), kvutil.CASConfig{
 		Attempts:       tagCASAttempts,
 		CreateIfAbsent: true,
 	}, data)

spinifex/domains/ec2/vpc/default_vpc.go

@@ -12,7 +12,7 @@ import (
 
 	networkv1 "github.com/mulgadc/spinifex/contracts/network/v1"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -105,7 +105,7 @@ func (s *VPCServiceImpl) ensureDefaultVPC(ctx context.Context, accountID string,
 			return claim.info(), nil
 		}
 
-		if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, claim.VpcId)); err == nil {
+		if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, claim.VpcId)); err == nil {
 			return claim.info(), nil
 		} else if !errors.Is(err, jetstream.ErrKeyNotFound) {
 			return nil, fmt.Errorf("read default VPC %s: %w", claim.VpcId, err)
@@ -233,7 +233,7 @@ func oldestAccountRecord[T any](ctx context.Context, kv jetstream.KeyValue, acco
 	var bestID string
 	var bestCreated time.Time
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := kv.Get(ctx, key)
@@ -262,7 +262,7 @@ func oldestAccountRecord[T any](ctx context.Context, kv jetstream.KeyValue, acco
 // marks the claim complete. Records are written with Create, so only the caller
 // that actually wrote one announces it to vpcd.
 func (s *VPCServiceImpl) buildDefaultVPC(ctx context.Context, accountID string, claim *defaultVPCClaim) error {
-	created, err := createRecord(ctx, s.vpcKV, kvstore.AccountKey(accountID, claim.VpcId), VPCRecord{
+	created, err := createRecord(ctx, s.vpcKV, kvutil.AccountKey(accountID, claim.VpcId), VPCRecord{
 		VpcId:              claim.VpcId,
 		CidrBlock:          claim.Cidr,
 		State:              "available",
@@ -284,7 +284,7 @@ func (s *VPCServiceImpl) buildDefaultVPC(ctx context.Context, accountID string,
 	if s.config != nil && s.config.AZ != "" {
 		az = s.config.AZ
 	}
-	created, err = createRecord(ctx, s.subnetKV, kvstore.AccountKey(accountID, claim.SubnetId), SubnetRecord{
+	created, err = createRecord(ctx, s.subnetKV, kvutil.AccountKey(accountID, claim.SubnetId), SubnetRecord{
 		SubnetId:            claim.SubnetId,
 		VpcId:               claim.VpcId,
 		CidrBlock:           claim.SubnetCidr,
@@ -320,7 +320,7 @@ func (s *VPCServiceImpl) buildDefaultVPC(ctx context.Context, accountID string,
 		}
 	}
 
-	if _, err := kvstore.Update(ctx, s.defaultsKV, accountID, kvstore.CASConfig{}, func(c *defaultVPCClaim) (bool, error) {
+	if _, err := kvutil.Update(ctx, s.defaultsKV, accountID, kvutil.CASConfig{}, func(c *defaultVPCClaim) (bool, error) {
 		if c.Complete || c.VpcId != claim.VpcId {
 			return false, nil
 		}

spinifex/domains/ec2/vpc/default_vpc_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
@@ -149,12 +149,12 @@ func TestEnsureDefaultVPC_AdoptsOldestExistingDefault(t *testing.T) {
 	const accountID = "000000000051"
 	created := time.Date(2026, 9, 11, 14, 13, 40, 0, time.UTC)
 	for i, id := range []string{"vpc-0000000000000newer", "vpc-0000000000000older"} {
-		putRecord(t, js, ec2vpc.KVBucketVPCs, kvstore.AccountKey(accountID, id), ec2vpc.VPCRecord{
+		putRecord(t, js, ec2vpc.KVBucketVPCs, kvutil.AccountKey(accountID, id), ec2vpc.VPCRecord{
 			VpcId: id, CidrBlock: ec2vpc.DefaultVPCCidr, IsDefault: true, VNI: int64(200 + i),
 			CreatedAt: created.Add(time.Duration(1-i) * time.Minute),
 		})
 	}
-	putRecord(t, js, ec2vpc.KVBucketSubnets, kvstore.AccountKey(accountID, "subnet-00000000000older"), ec2vpc.SubnetRecord{
+	putRecord(t, js, ec2vpc.KVBucketSubnets, kvutil.AccountKey(accountID, "subnet-00000000000older"), ec2vpc.SubnetRecord{
 		SubnetId: "subnet-00000000000older", VpcId: "vpc-0000000000000older", CidrBlock: ec2vpc.DefaultSubnetCidr,
 		IsDefault: true, CreatedAt: created,
 	})
@@ -183,7 +183,7 @@ func TestEnsureDefaultVPC_FinishesHalfBuiltClaim(t *testing.T) {
 		"route_table_id": "rtb-00000000000half", "internet_gateway_id": "igw-00000000000half",
 		"cidr": ec2vpc.DefaultVPCCidr, "subnet_cidr": ec2vpc.DefaultSubnetCidr, "vni": 4242,
 	})
-	putRecord(t, js, ec2vpc.KVBucketVPCs, kvstore.AccountKey(accountID, "vpc-00000000000000half"), ec2vpc.VPCRecord{
+	putRecord(t, js, ec2vpc.KVBucketVPCs, kvutil.AccountKey(accountID, "vpc-00000000000000half"), ec2vpc.VPCRecord{
 		VpcId: "vpc-00000000000000half", CidrBlock: ec2vpc.DefaultVPCCidr, IsDefault: true, VNI: 4242,
 	})
 

spinifex/domains/ec2/vpc/eni.go

@@ -23,7 +23,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -118,7 +118,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2.
 	subnetId := *input.SubnetId
 
 	// Verify subnet exists and belongs to this account
-	subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, subnetId))
+	subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, subnetId))
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound)
 	}
@@ -205,7 +205,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2.
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal ENI record: %w", err)
 	}
-	if _, err := s.eniKV.Put(ctx, kvstore.AccountKey(accountID, eniId), data); err != nil {
+	if _, err := s.eniKV.Put(ctx, kvutil.AccountKey(accountID, eniId), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -251,7 +251,7 @@ func (s *VPCServiceImpl) ForceDeleteInstanceENI(ctx context.Context, accountID,
 }
 
 func (s *VPCServiceImpl) deleteNetworkInterface(ctx context.Context, eniId, accountID string, force bool) (*ec2.DeleteNetworkInterfaceOutput, error) {
-	key := kvstore.AccountKey(accountID, eniId)
+	key := kvutil.AccountKey(accountID, eniId)
 
 	// Get the ENI record
 	entry, err := s.eniKV.Get(ctx, key)
@@ -346,7 +346,7 @@ func (s *VPCServiceImpl) releaseENISideEffects(ctx context.Context, eniId, accou
 // deletion that did not happen.
 func (s *VPCServiceImpl) DetachAndDeleteENI(ctx context.Context, accountID, eniID string, force bool) (deleted bool, err error) {
 	const maxAttempts = 3
-	key := kvstore.AccountKey(accountID, eniID)
+	key := kvutil.AccountKey(accountID, eniID)
 	var lastErr error
 
 	for range maxAttempts {
@@ -458,7 +458,7 @@ func (s *VPCServiceImpl) ModifyNetworkInterfaceAttribute(ctx context.Context, in
 	}
 
 	eniId := *input.NetworkInterfaceId
-	key := kvstore.AccountKey(accountID, eniId)
+	key := kvutil.AccountKey(accountID, eniId)
 
 	entry, err := s.eniKV.Get(ctx, key)
 	if err != nil {
@@ -569,7 +569,7 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *e
 	}
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey {
+		if key == kvutil.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -707,7 +707,7 @@ func (s *VPCServiceImpl) AttachENIWithOwner(accountID, eniId, instanceId string,
 
 func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instanceId string, deviceIndex int64, instanceOwnerID string) (string, error) {
 	attachmentId := awsidentifiers.GenerateResourceID("eni-attach")
-	err := s.casENI(ctx, kvstore.AccountKey(accountID, eniId), eniId, func(record *ENIRecord) error {
+	err := s.casENI(ctx, kvutil.AccountKey(accountID, eniId), eniId, func(record *ENIRecord) error {
 		if record.Status == "in-use" {
 			return errors.New(awserrors.ErrorInvalidNetworkInterfaceInUse)
 		}
@@ -737,7 +737,7 @@ func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instan
 // DetachENI marks an ENI as detached from an instance (internal use by TerminateInstances).
 // accountID scopes the lookup to the correct KV key.
 func (s *VPCServiceImpl) DetachENI(ctx context.Context, accountID, eniId string) error {
-	key := kvstore.AccountKey(accountID, eniId)
+	key := kvutil.AccountKey(accountID, eniId)
 	entry, err := s.eniKV.Get(ctx, key)
 	if err != nil {
 		return eniNotFoundError(eniId)
@@ -773,7 +773,7 @@ func (s *VPCServiceImpl) GetENIRecord(accountID, eniId string) (ENIRecord, error
 }
 
 func (s *VPCServiceImpl) getENIRecord(ctx context.Context, accountID, eniId string) (ENIRecord, error) {
-	key := kvstore.AccountKey(accountID, eniId)
+	key := kvutil.AccountKey(accountID, eniId)
 	entry, err := s.eniKV.Get(ctx, key)
 	if err != nil {
 		return ENIRecord{}, eniNotFoundError(eniId)
@@ -793,7 +793,7 @@ func (s *VPCServiceImpl) UpdateENI(accountID, eniId string, fn func(*ENIRecord))
 }
 
 func (s *VPCServiceImpl) updateENI(ctx context.Context, accountID, eniId string, fn func(*ENIRecord)) error {
-	return s.casENI(ctx, kvstore.AccountKey(accountID, eniId), eniId, func(r *ENIRecord) error {
+	return s.casENI(ctx, kvutil.AccountKey(accountID, eniId), eniId, func(r *ENIRecord) error {
 		fn(r)
 		return nil
 	})
@@ -876,7 +876,7 @@ func (s *VPCServiceImpl) listInstanceENIs(ctx context.Context, accountID, instan
 	prefix := accountID + "."
 	var out []ENIRecord
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, key)
@@ -938,7 +938,7 @@ func (s *VPCServiceImpl) scanENIs(ctx context.Context, match func(*ENIRecord) bo
 
 	var out []AccountENI
 	for _, key := range keys {
-		if key == kvstore.VersionKey {
+		if key == kvutil.VersionKey {
 			continue
 		}
 		accountID, _, found := strings.Cut(key, ".")
@@ -1004,7 +1004,7 @@ func (s *VPCServiceImpl) findENIByAttachment(ctx context.Context, accountID, att
 	}
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, key)
@@ -1028,7 +1028,7 @@ func (s *VPCServiceImpl) UpdateENIPublicIP(accountID, eniId, publicIP, poolName
 }
 
 func (s *VPCServiceImpl) updateENIPublicIP(ctx context.Context, accountID, eniId, publicIP, poolName string) error {
-	key := kvstore.AccountKey(accountID, eniId)
+	key := kvutil.AccountKey(accountID, eniId)
 	entry, err := s.eniKV.Get(ctx, key)
 	if err != nil {
 		return fmt.Errorf("ENI %s not found: %w", eniId, err)
@@ -1063,7 +1063,7 @@ func (s *VPCServiceImpl) fillSGNames(ctx context.Context, accountID string, ids
 			continue
 		}
 		names[id] = ""
-		entry, err := s.sgKV.Get(ctx, kvstore.AccountKey(accountID, id))
+		entry, err := s.sgKV.Get(ctx, kvutil.AccountKey(accountID, id))
 		if err != nil {
 			slog.WarnContext(ctx, "fillSGNames: SG read failed", "groupId", id, "err", err)
 			continue
@@ -1080,7 +1080,7 @@ func (s *VPCServiceImpl) fillSGNames(ctx context.Context, accountID string, ids
 // vpcDNSHostnames reports whether the VPC has enableDnsHostnames on. An
 // unreadable VPC reads as off, so its interfaces go unnamed.
 func (s *VPCServiceImpl) vpcDNSHostnames(ctx context.Context, accountID, vpcID string) bool {
-	entry, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID))
+	entry, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID))
 	if err != nil {
 		slog.WarnContext(ctx, "vpcDNSHostnames: VPC read failed", "vpcId", vpcID, "err", err)
 		return false
@@ -1238,7 +1238,7 @@ func (s *VPCServiceImpl) validateSGAttachment(ctx context.Context, accountID str
 
 	// Each SG must exist in the caller's account and belong to the same VPC.
 	for _, sgId := range sgIds {
-		sgEntry, err := s.sgKV.Get(ctx, kvstore.AccountKey(accountID, sgId))
+		sgEntry, err := s.sgKV.Get(ctx, kvutil.AccountKey(accountID, sgId))
 		if err != nil {
 			return sgNotFoundError(sgId)
 		}

spinifex/domains/ec2/vpc/eni_lease_rebind.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/domains/network/projection"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/topology"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -69,7 +69,7 @@ func (s *VPCServiceImpl) findENIAnyAccount(ctx context.Context, eniID string) (*
 
 	suffix := "." + eniID
 	for _, k := range keys {
-		if k == kvstore.VersionKey || !strings.HasSuffix(k, suffix) {
+		if k == kvutil.VersionKey || !strings.HasSuffix(k, suffix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, k)

spinifex/domains/ec2/vpc/eni_stale_read_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -30,7 +30,7 @@ func newStaleReadENI(t *testing.T) (*VPCServiceImpl, string, func() ENIRecord) {
 	// Reads the stored record directly: the wrapped bucket would serve the
 	// check itself from a lagging replica.
 	stored := func() ENIRecord {
-		entry, err := inner.Get(context.Background(), kvstore.AccountKey(testAccountID, eniID))
+		entry, err := inner.Get(context.Background(), kvutil.AccountKey(testAccountID, eniID))
 		require.NoError(t, err)
 		var rec ENIRecord
 		require.NoError(t, json.Unmarshal(entry.Value(), &rec))
@@ -114,7 +114,7 @@ func TestCASENIGivesUpAfterBoundedConflicts(t *testing.T) {
 	t.Parallel()
 	svc, eniID, _ := newStaleReadENI(t)
 	conflicts := 0
-	err := svc.casENI(t.Context(), kvstore.AccountKey(testAccountID, eniID), eniID, func(*ENIRecord) error {
+	err := svc.casENI(t.Context(), kvutil.AccountKey(testAccountID, eniID), eniID, func(*ENIRecord) error {
 		conflicts++
 		// Move the record under the next compare-and-swap, as a competing writer does.
 		return svc.UpdateENI(testAccountID, eniID, func(r *ENIRecord) { r.Description = "other writer" })

spinifex/domains/ec2/vpc/lease_owner.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -42,7 +42,7 @@ func keyWithSuffixExists(ctx context.Context, kv jetstream.KeyValue, resourceID
 	}
 	suffix := "." + resourceID
 	for _, k := range keys {
-		if k != kvstore.VersionKey && strings.HasSuffix(k, suffix) {
+		if k != kvutil.VersionKey && strings.HasSuffix(k, suffix) {
 			return true, nil
 		}
 	}

spinifex/domains/ec2/vpc/security_group.go

@@ -23,7 +23,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -245,7 +245,7 @@ func (s *VPCServiceImpl) CreateSecurityGroup(ctx context.Context, input *ec2.Cre
 	}
 	sgsInVPC := 0
 	for _, k := range sgKeys {
-		if k == kvstore.VersionKey {
+		if k == kvutil.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -305,7 +305,7 @@ func (s *VPCServiceImpl) CreateSecurityGroup(ctx context.Context, input *ec2.Cre
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal security group record: %w", err)
 	}
-	if _, err := s.sgKV.Put(ctx, kvstore.AccountKey(accountID, groupId), data); err != nil {
+	if _, err := s.sgKV.Put(ctx, kvutil.AccountKey(accountID, groupId), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -331,7 +331,7 @@ func (s *VPCServiceImpl) DeleteSecurityGroup(ctx context.Context, input *ec2.Del
 	}
 
 	groupId := *input.GroupId
-	key := kvstore.AccountKey(accountID, groupId)
+	key := kvutil.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -380,7 +380,7 @@ func (s *VPCServiceImpl) validateSGRuleReferences(ctx context.Context, accountID
 		if r.SourceSG == "" {
 			continue
 		}
-		entry, err := s.sgKV.Get(ctx, kvstore.AccountKey(accountID, r.SourceSG))
+		entry, err := s.sgKV.Get(ctx, kvutil.AccountKey(accountID, r.SourceSG))
 		if err != nil {
 			return sgNotFoundError(r.SourceSG)
 		}
@@ -406,7 +406,7 @@ func (s *VPCServiceImpl) checkSGDependencies(ctx context.Context, accountID, gro
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range eniKeys {
-		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, k)
@@ -435,7 +435,7 @@ func (s *VPCServiceImpl) checkSGDependencies(ctx context.Context, accountID, gro
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range sgKeys {
-		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, k)
@@ -561,7 +561,7 @@ func (s *VPCServiceImpl) DescribeSecurityGroups(ctx context.Context, input *ec2.
 	}
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey {
+		if key == kvutil.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -713,7 +713,7 @@ func (s *VPCServiceImpl) GetSecurityGroupsForVpc(ctx context.Context, input *ec2
 
 	groups := []*ec2.SecurityGroupForVpc{}
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, key)
@@ -844,7 +844,7 @@ func (s *VPCServiceImpl) DescribeSecurityGroupRules(ctx context.Context, input *
 	emitted := make(map[string]bool)
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, key)
@@ -945,7 +945,7 @@ func (s *VPCServiceImpl) updateSGRuleTags(ctx context.Context, accountID, ruleID
 	}
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, key)
@@ -1056,7 +1056,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupIngress(ctx context.Context, inpu
 	}
 
 	groupId := *input.GroupId
-	key := kvstore.AccountKey(accountID, groupId)
+	key := kvutil.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -1134,7 +1134,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupEgress(ctx context.Context, input
 	}
 
 	groupId := *input.GroupId
-	key := kvstore.AccountKey(accountID, groupId)
+	key := kvutil.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -1212,7 +1212,7 @@ func (s *VPCServiceImpl) RevokeSecurityGroupIngress(ctx context.Context, input *
 	}
 
 	groupId := *input.GroupId
-	key := kvstore.AccountKey(accountID, groupId)
+	key := kvutil.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -1277,7 +1277,7 @@ func (s *VPCServiceImpl) RevokeSecurityGroupEgress(ctx context.Context, input *e
 	}
 
 	groupId := *input.GroupId
-	key := kvstore.AccountKey(accountID, groupId)
+	key := kvutil.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -1397,7 +1397,7 @@ func (s *VPCServiceImpl) updateSGRuleDescriptions(ctx context.Context, accountID
 		return errors.New(awserrors.ErrorInvalidParameterCombination)
 	}
 
-	key := kvstore.AccountKey(accountID, req.groupId)
+	key := kvutil.AccountKey(accountID, req.groupId)
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
 		return sgNotFoundError(req.groupId)
@@ -1581,7 +1581,7 @@ func (s *VPCServiceImpl) ModifySecurityGroupRules(ctx context.Context, input *ec
 	}
 
 	groupId := *input.GroupId
-	key := kvstore.AccountKey(accountID, groupId)
+	key := kvutil.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -2158,7 +2158,7 @@ func (s *VPCServiceImpl) storeDefaultSecurityGroup(ctx context.Context, accountI
 	if err != nil {
 		return nil, false, fmt.Errorf("marshal default security group: %w", err)
 	}
-	if _, err := s.sgKV.Create(ctx, kvstore.AccountKey(accountID, groupId), data); err != nil {
+	if _, err := s.sgKV.Create(ctx, kvutil.AccountKey(accountID, groupId), data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
 			return record, false, nil
 		}
@@ -2182,7 +2182,7 @@ func (s *VPCServiceImpl) announceDefaultSecurityGroup(record *SecurityGroupRecor
 // CannotDelete guard for default SGs. Used by DeleteVpc to cascade-delete the
 // per-VPC default SG. Surfaces vpcd tear-down failures to the caller.
 func (s *VPCServiceImpl) deleteSecurityGroupInternal(ctx context.Context, accountID, groupId string) error {
-	key := kvstore.AccountKey(accountID, groupId)
+	key := kvutil.AccountKey(accountID, groupId)
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
 		return fmt.Errorf("read default security group: %w", err)
@@ -2219,7 +2219,7 @@ func (s *VPCServiceImpl) findDefaultSGForVPC(ctx context.Context, accountID, vpc
 		return "", err
 	}
 	for _, k := range keys {
-		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, k)

spinifex/domains/ec2/vpc/security_group_modify_test.go

@@ -15,7 +15,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -62,7 +62,7 @@ func tcpPeerRequest(port int64, peer string) *ec2.SecurityGroupRuleRequest {
 // modify wrote nothing.
 func storedSGRecord(t *testing.T, svc *VPCServiceImpl, sgID string) []byte {
 	t.Helper()
-	entry, err := svc.sgKV.Get(t.Context(), kvstore.AccountKey(testAccountID, sgID))
+	entry, err := svc.sgKV.Get(t.Context(), kvutil.AccountKey(testAccountID, sgID))
 	require.NoError(t, err)
 	return entry.Value()
 }
@@ -689,7 +689,7 @@ func TestRevokeSecurityGroupIngress_InvertedPortRangeStillRevocable(t *testing.T
 	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
 	sgID := createTestSG(t, svc, vpcID, "stale-inverted-ports")
 
-	key := kvstore.AccountKey(testAccountID, sgID)
+	key := kvutil.AccountKey(testAccountID, sgID)
 	var rec SecurityGroupRecord
 	require.NoError(t, json.Unmarshal(storedSGRecord(t, svc, sgID), &rec))
 	rec.IngressRules = append(rec.IngressRules, SGRule{

spinifex/domains/ec2/vpc/security_group_rule_validation_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -35,7 +35,7 @@ func plantIngressRules(t *testing.T, svc *VPCServiceImpl, sgID string, rules ...
 	rec.IngressRules = append(rec.IngressRules, rules...)
 	data, err := json.Marshal(rec)
 	require.NoError(t, err)
-	_, err = svc.sgKV.Put(t.Context(), kvstore.AccountKey(testAccountID, sgID), data)
+	_, err = svc.sgKV.Put(t.Context(), kvutil.AccountKey(testAccountID, sgID), data)
 	require.NoError(t, err)
 }
 

spinifex/domains/ec2/vpc/security_group_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -1524,7 +1524,7 @@ func findDefaultSGInVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) string
 	require.NoError(t, err)
 	var matches []string
 	for _, k := range keys {
-		if k == kvstore.VersionKey {
+		if k == kvutil.VersionKey {
 			continue
 		}
 		entry, err := svc.sgKV.Get(t.Context(), k)
@@ -2122,7 +2122,7 @@ func seedAvailableVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) {
 	rec := VPCRecord{VpcId: vpcID, CidrBlock: "10.0.0.0/16", State: "available"}
 	data, err := json.Marshal(rec)
 	require.NoError(t, err)
-	_, err = svc.vpcKV.Put(t.Context(), kvstore.AccountKey(testAccountID, vpcID), data)
+	_, err = svc.vpcKV.Put(t.Context(), kvutil.AccountKey(testAccountID, vpcID), data)
 	require.NoError(t, err)
 }
 
@@ -2359,7 +2359,7 @@ func TestAuthorizeSecurityGroupIngress_AssignsRuleIDs(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 
-	entry, err := svc.sgKV.Get(t.Context(), kvstore.AccountKey(testAccountID, sgID))
+	entry, err := svc.sgKV.Get(t.Context(), kvutil.AccountKey(testAccountID, sgID))
 	require.NoError(t, err)
 	var record SecurityGroupRecord
 	require.NoError(t, json.Unmarshal(entry.Value(), &record))

spinifex/domains/ec2/vpc/service_impl.go

@@ -26,7 +26,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/zoneid"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -173,7 +173,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("create JetStream client: %w", err)
 	}
 
-	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketVPCs, 10)
+	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketVPCs, err)
 	}
@@ -181,7 +181,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketVPCs, err)
 	}
 
-	subnetKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketSubnets, 10)
+	subnetKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketSubnets, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSubnets, err)
 	}
@@ -189,7 +189,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketSubnets, err)
 	}
 
-	vniKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketVNICounter, 10)
+	vniKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketVNICounter, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketVNICounter, err)
 	}
@@ -197,7 +197,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketVNICounter, err)
 	}
 
-	eniKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketENIs, 10)
+	eniKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketENIs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketENIs, err)
 	}
@@ -205,7 +205,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketENIs, err)
 	}
 
-	sgKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketSecurityGroups, 10)
+	sgKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketSecurityGroups, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSecurityGroups, err)
 	}
@@ -213,7 +213,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketSecurityGroups, err)
 	}
 
-	defaultsKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketDefaultVPCs, 10)
+	defaultsKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketDefaultVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketDefaultVPCs, err)
 	}
@@ -221,18 +221,18 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketDefaultVPCs, err)
 	}
 
-	rtbKV, err := kvstore.GetOrCreateBucket(ctx, js, "spinifex-vpc-route-tables", 10)
+	rtbKV, err := kvutil.GetOrCreateBucket(ctx, js, "spinifex-vpc-route-tables", 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket spinifex-vpc-route-tables: %w", err)
 	}
 
 	// Named literally, like the route table bucket above: the IGW package
 	// imports this one, so its constant cannot be referenced here.
-	igwKV, err := kvstore.GetOrCreateBucket(ctx, js, "spinifex-igw", 10)
+	igwKV, err := kvutil.GetOrCreateBucket(ctx, js, "spinifex-igw", 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket spinifex-igw: %w", err)
 	}
-	eigwKV, err := kvstore.GetOrCreateBucket(ctx, js, "spinifex-eigw", 10)
+	eigwKV, err := kvutil.GetOrCreateBucket(ctx, js, "spinifex-eigw", 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket spinifex-eigw: %w", err)
 	}
@@ -268,7 +268,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 // nextVNI allocates the next VNI using atomic increment on the NATS KV counter.
 func (s *VPCServiceImpl) nextVNI(ctx context.Context) (int64, error) {
 	var allocated int64
-	_, err := kvstore.Update(ctx, s.vniKV, vniCounterKey, kvstore.CASConfig{CreateIfAbsent: true, Attempts: vniCASAttempts},
+	_, err := kvutil.Update(ctx, s.vniKV, vniCounterKey, kvutil.CASConfig{CreateIfAbsent: true, Attempts: vniCASAttempts},
 		func(next *int64) (bool, error) {
 			if *next == 0 {
 				*next = vniStart
@@ -331,7 +331,7 @@ func (s *VPCServiceImpl) CreateVpc(ctx context.Context, input *ec2.CreateVpcInpu
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal VPC record: %w", err)
 	}
-	if _, err := s.vpcKV.Put(ctx, kvstore.AccountKey(accountID, vpcID), data); err != nil {
+	if _, err := s.vpcKV.Put(ctx, kvutil.AccountKey(accountID, vpcID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -366,7 +366,7 @@ func (s *VPCServiceImpl) CreateVpc(ctx context.Context, input *ec2.CreateVpcInpu
 // requireVPCExists returns InvalidVpcID.NotFound if the VPC doesn't exist for
 // this account.
 func (s *VPCServiceImpl) requireVPCExists(ctx context.Context, accountID, vpcId string) error {
-	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcId)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcId)); err != nil {
 		return vpcNotFoundError(vpcId)
 	}
 	return nil
@@ -379,7 +379,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 	}
 
 	vpcID := *input.VpcId
-	key := kvstore.AccountKey(accountID, vpcID)
+	key := kvutil.AccountKey(accountID, vpcID)
 
 	if _, err := s.vpcKV.Get(ctx, key); err != nil {
 		// AWS-faithful: an absent VPC is NotFound (the tofu/SDK provider
@@ -398,7 +398,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range subnetKeys {
-		if k == kvstore.VersionKey {
+		if k == kvutil.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -435,7 +435,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range sgKeys {
-		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, k)
@@ -472,7 +472,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range rtbKeys {
-		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.rtbKV.Get(ctx, k)
@@ -533,7 +533,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	if rtbID != "" {
-		if err := s.rtbKV.Delete(ctx, kvstore.AccountKey(accountID, rtbID)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
+		if err := s.rtbKV.Delete(ctx, kvutil.AccountKey(accountID, rtbID)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
 			slog.ErrorContext(ctx, "DeleteVpc: main route table reap failed", "vpcId", vpcID, "routeTableId", rtbID, "err", err)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
@@ -574,7 +574,7 @@ func rejectAttachedGateway(ctx context.Context, kv jetstream.KeyValue, accountID
 	}
 
 	for _, k := range keys {
-		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := kv.Get(ctx, k)
@@ -653,7 +653,7 @@ func (s *VPCServiceImpl) DescribeVpcs(ctx context.Context, input *ec2.DescribeVp
 	}
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey {
+		if key == kvutil.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -723,7 +723,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
 	}
 
 	// Verify VPC exists and belongs to this account
-	vpcEntry, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID))
+	vpcEntry, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID))
 	if err != nil {
 		return nil, vpcNotFoundError(vpcID)
 	}
@@ -757,7 +757,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
 	}
 
 	for _, k := range subnetKeys {
-		if k == kvstore.VersionKey {
+		if k == kvutil.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -822,7 +822,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal subnet record: %w", err)
 	}
-	if _, err := s.subnetKV.Put(ctx, kvstore.AccountKey(accountID, subnetID), data); err != nil {
+	if _, err := s.subnetKV.Put(ctx, kvutil.AccountKey(accountID, subnetID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -843,7 +843,7 @@ func (s *VPCServiceImpl) DeleteSubnet(ctx context.Context, input *ec2.DeleteSubn
 	}
 
 	subnetID := *input.SubnetId
-	key := kvstore.AccountKey(accountID, subnetID)
+	key := kvutil.AccountKey(accountID, subnetID)
 
 	// Read subnet record before deletion (needed for vpcd event)
 	subnetEntry, err := s.subnetKV.Get(ctx, key)
@@ -902,7 +902,7 @@ func (s *VPCServiceImpl) checkSubnetResidents(ctx context.Context, accountID, su
 	}
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, key)
@@ -950,7 +950,7 @@ func (s *VPCServiceImpl) clearRouteTableAssociationsForSubnet(ctx context.Contex
 	}
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.rtbKV.Get(ctx, key)
@@ -1035,7 +1035,7 @@ func (s *VPCServiceImpl) DescribeSubnets(ctx context.Context, input *ec2.Describ
 	}
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey {
+		if key == kvutil.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -1386,7 +1386,7 @@ func (s *VPCServiceImpl) ModifySubnetAttribute(ctx context.Context, input *ec2.M
 	}
 
 	subnetID := *input.SubnetId
-	key := kvstore.AccountKey(accountID, subnetID)
+	key := kvutil.AccountKey(accountID, subnetID)
 
 	entry, err := s.subnetKV.Get(ctx, key)
 	if err != nil {
@@ -1438,7 +1438,7 @@ func (s *VPCServiceImpl) ModifyVpcAttribute(ctx context.Context, input *ec2.Modi
 	}
 
 	vpcID := *input.VpcId
-	key := kvstore.AccountKey(accountID, vpcID)
+	key := kvutil.AccountKey(accountID, vpcID)
 
 	entry, err := s.vpcKV.Get(ctx, key)
 	if err != nil {
@@ -1486,7 +1486,7 @@ func (s *VPCServiceImpl) DescribeVpcAttribute(ctx context.Context, input *ec2.De
 	}
 
 	vpcID := *input.VpcId
-	key := kvstore.AccountKey(accountID, vpcID)
+	key := kvutil.AccountKey(accountID, vpcID)
 
 	entry, err := s.vpcKV.Get(ctx, key)
 	if err != nil {
@@ -1587,7 +1587,7 @@ func (s *VPCServiceImpl) writeMainRouteTable(ctx context.Context, accountID, vpc
 	if err != nil {
 		return false, fmt.Errorf("marshal main route table: %w", err)
 	}
-	if _, err := s.rtbKV.Create(ctx, kvstore.AccountKey(accountID, rtbID), data); err != nil {
+	if _, err := s.rtbKV.Create(ctx, kvutil.AccountKey(accountID, rtbID), data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
 			return false, nil
 		}
@@ -1611,7 +1611,7 @@ func (s *VPCServiceImpl) findMainRouteTableID(ctx context.Context, accountID, vp
 		return "", fmt.Errorf("list rtb keys: %w", err)
 	}
 	for _, key := range keys {
-		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.rtbKV.Get(ctx, key)
@@ -1652,7 +1652,7 @@ func (s *VPCServiceImpl) getDefaultSubnet(ctx context.Context, accountID string)
 	}
 
 	for _, key := range keys {
-		if key == kvstore.VersionKey {
+		if key == kvutil.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -1680,7 +1680,7 @@ func (s *VPCServiceImpl) GetSubnet(accountID, subnetId string) (*SubnetRecord, e
 }
 
 func (s *VPCServiceImpl) getSubnet(ctx context.Context, accountID, subnetId string) (*SubnetRecord, error) {
-	key := kvstore.AccountKey(accountID, subnetId)
+	key := kvutil.AccountKey(accountID, subnetId)
 	entry, err := s.subnetKV.Get(ctx, key)
 	if err != nil {
 		return nil, fmt.Errorf("subnet %s not found: %w", subnetId, err)
@@ -1726,7 +1726,7 @@ func (s *VPCServiceImpl) SubnetAvailabilityZones(ctx context.Context) ([]string,
 	}
 	seen := map[string]struct{}{}
 	for _, k := range keys {
-		if k == kvstore.VersionKey {
+		if k == kvutil.VersionKey {
 			continue
 		}
 		entry, err := s.subnetKV.Get(ctx, k)

spinifex/domains/ec2/vpc/service_impl_test.go

@@ -16,7 +16,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/zoneid"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
@@ -2689,7 +2689,7 @@ func TestCreateSubnet_ZoneStoreFailureWritesNothingAndPublishesNothing(t *testin
 		require.ErrorIs(t, err, jetstream.ErrNoKeysFound)
 	}
 	for _, k := range keys {
-		assert.Equal(t, kvstore.VersionKey, k, "no subnet record may exist")
+		assert.Equal(t, kvutil.VersionKey, k, "no subnet record may exist")
 	}
 	assert.Zero(t, events.Load(), "no topology event may be published")
 }

spinifex/domains/ecr/auth/ecrauth_test.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/golang-jwt/jwt/v5"
 	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -111,7 +111,7 @@ func TestLoadOrCreateSigningKey_FailsWhenNoKeyDecrypts(t *testing.T) {
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "master key does not match")
 
-	entries, err := kvstore.Keys(t.Context(), kv)
+	entries, err := kvutil.Keys(t.Context(), kv)
 	require.NoError(t, err)
 	assert.Len(t, entries, 1, "a failed load must not mint a key alongside the unreadable one")
 }

spinifex/domains/ecr/auth/signingkey.go

@@ -21,7 +21,7 @@ import (
 	"time"
 
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -111,7 +111,7 @@ func openSigningBucket(ctx context.Context, js jetstream.JetStream, masterKey []
 	if len(masterKey) == 0 {
 		return nil, errors.New("ecrauth: empty master key")
 	}
-	return kvstore.GetOrCreateBucket(ctx, js, SigningBucket, signingKeyHistory)
+	return kvutil.GetOrCreateBucket(ctx, js, SigningBucket, signingKeyHistory)
 }
 
 // reloadKeys reads every stored signing key, returning the verify set, each
@@ -125,7 +125,7 @@ func openSigningBucket(ctx context.Context, js jetstream.JetStream, masterKey []
 // load instead takes the gateway down on every node, permanently, over a key no
 // live request needs.
 func reloadKeys(ctx context.Context, kv jetstream.KeyValue, masterKey []byte) (active *SigningKey, verify map[string]*ecdsa.PublicKey, metas []keyMeta, undecodable int, err error) {
-	names, err := kvstore.Keys(ctx, kv)
+	names, err := kvutil.Keys(ctx, kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, nil, nil, 0, fmt.Errorf("list signing keys: %w", err)
 	}

spinifex/domains/ecr/meta.go

@@ -10,7 +10,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -179,7 +179,7 @@ func (s *KVMetaStore) bucket(ctx context.Context, accountID string) (jetstream.K
 		return kv, nil
 	}
 	name := KVAccountBucket(accountID)
-	kv, err := kvstore.GetOrCreateBucket(ctx, s.js, name, KVBucketAccountHistory)
+	kv, err := kvutil.GetOrCreateBucket(ctx, s.js, name, KVBucketAccountHistory)
 	if err != nil {
 		return nil, fmt.Errorf("ecr: open account bucket %s: %w", name, err)
 	}

spinifex/domains/ecs/eni_test.go

@@ -322,7 +322,7 @@ func TestReaper_Awsvpc_ReleasesENI(t *testing.T) {
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
-	sc := testScheduler(t, nc, svc, "test-holder")
+	sc := NewScheduler(nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	assert.Equal(t, 1, eni.releaseCalls)

spinifex/domains/ecs/enumeration_failure_test.go

@@ -27,7 +27,7 @@ func dropRevisit(pass func(context.Context) (time.Duration, error)) func(context
 // drained, services never relaunched, stopped tasks never pruned.
 func TestSchedulerPasses_EnumerationFailureIsReported(t *testing.T) {
 	svc, nc := newTestService(t)
-	sc := testScheduler(t, nc, svc, "test-holder")
+	sc := NewScheduler(nc, svc, "test-holder")
 	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
 	require.NoError(t, err)
 

spinifex/domains/ecs/instance_recovery_test.go

@@ -49,7 +49,7 @@ func TestRegister_RestoresReapedInstanceToActive(t *testing.T) {
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
-	testScheduler(t, nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
+	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	reaped := instanceStatus(t, svc, "web", "i-1")
 	require.Equal(t, InstanceStatusDraining, reaped.Status)
@@ -89,7 +89,7 @@ func TestReaper_ReleasesCapacityAndRecoversClean(t *testing.T) {
 	seeded.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &seeded))
 
-	testScheduler(t, nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
+	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	reaped := instanceStatus(t, svc, "web", "i-1")
 	require.Equal(t, InstanceStatusDraining, reaped.Status)

spinifex/domains/ecs/revisit_test.go

@@ -50,7 +50,7 @@ func schedulerRig(t *testing.T) (*Scheduler, *Service, jetstream.KeyValue) {
 	registerInstance(t, svc, "web", "i-1", 4096, 8192)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	return testScheduler(t, nc, svc, "test-holder"), svc, kv
+	return NewScheduler(nc, svc, "test-holder"), svc, kv
 }
 
 // runOneTask places a task and returns its ID, read back from the record rather

spinifex/domains/ecs/scheduler.go

@@ -13,19 +13,17 @@ import (
 
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
-// KeyScheduler is the ECS scheduler's leader lease in the sync bucket.
-const KeyScheduler = "ecs.scheduler"
-
-// Scheduler timing. A follower asks again at the lease's renewal rate. The
-// reaper marks an instance DRAINING after three missed 30s heartbeats.
+// Scheduler timing. The leader lease lives in spinifex-ecs-leader (60s TTL); the
+// holder refreshes well inside that. The reaper marks an instance DRAINING after
+// three missed 30s heartbeats.
 const (
-	schedulerLeaseTTL   = 60 * time.Second
-	leaseRefresh        = schedulerLeaseTTL / 3
+	schedulerLeaderKey  = "scheduler"
+	leaseRefresh        = 20 * time.Second
 	reaperInterval      = 30 * time.Second
 	heartbeatTimeout    = 90 * time.Second
 	stoppedReasonReaped = "ContainerInstance disconnected"
@@ -61,14 +59,17 @@ const (
 	reconcileResync = 5 * time.Minute
 )
 
-// Scheduler is the per-daemon ECS control loop. A single leader (elected on its
-// lease) owns the Layer-2 bus subscriptions and the heartbeat
+// Scheduler is the per-daemon ECS control loop. A single leader (elected via the
+// shared leader bucket) owns the Layer-2 bus subscriptions and the heartbeat
 // reaper; losers idle and retry. Placement itself happens synchronously in the
 // RunTask handler, so a brief leaderless gap never blocks RunTask.
 type Scheduler struct {
-	nc    *nats.Conn
-	svc   *Service
-	lease *kvsync.Lease
+	nc     *nats.Conn
+	svc    *Service
+	holder string
+
+	lease    *kvlease.Lease
+	leaseErr error
 
 	// wake carries a bus message that changed something the service reconcile
 	// reads. Fed by the task-state subscription and not by the heartbeat one:
@@ -88,28 +89,47 @@ type Scheduler struct {
 	subs []*nats.Subscription
 }
 
-// NewSchedulerLease returns the ECS scheduler's leader lease for holder.
-func NewSchedulerLease(store kvsync.KVStore, holder string) *kvsync.Lease {
-	return kvsync.NewLease(store, KeyScheduler, holder, schedulerLeaseTTL)
-}
-
-// NewScheduler constructs a Scheduler that leads while it holds lease.
-func NewScheduler(nc *nats.Conn, svc *Service, lease *kvsync.Lease) *Scheduler {
+// NewScheduler constructs a Scheduler. holder identifies this daemon in the lease.
+func NewScheduler(nc *nats.Conn, svc *Service, holder string) *Scheduler {
 	// One slot: it means "something the reconcile reads has changed", and a
 	// second arriving before the pass runs says nothing the first did not.
-	return &Scheduler{nc: nc, svc: svc, lease: lease, wake: make(chan struct{}, 1)}
+	sc := &Scheduler{nc: nc, svc: svc, holder: holder, wake: make(chan struct{}, 1)}
+	sc.lease, sc.leaseErr = kvlease.New(kvlease.Config{
+		Name:   "ecs/scheduler",
+		Bucket: sc.leaderBucket,
+		Key:    schedulerLeaderKey,
+		Holder: holder,
+		TTL:    KVBucketECSLeaderTTL,
+		Renew:  leaseRefresh,
+		Retry:  leaseRefresh,
+		// The leader owns the Layer-2 bus subscriptions and the instance-role
+		// converge pass. A node that can do neither stands down rather than hold
+		// the lease and process nothing.
+		OnGained: sc.onGained,
+		OnLost:   sc.unsubscribeBus,
+	})
+	return sc
+}
+
+func (sc *Scheduler) leaderBucket(ctx context.Context) (jetstream.KeyValue, error) {
+	js, err := jetstream.New(sc.nc)
+	if err != nil {
+		return nil, err
+	}
+	return InitLeaderBucket(ctx, js)
 }
 
 // Run drives the leadership + reaper loop until ctx is cancelled. It is intended
 // to run as a daemon-boot goroutine; panics are the caller's recover concern.
 func (sc *Scheduler) Run(ctx context.Context) {
+	if sc.leaseErr != nil {
+		slog.ErrorContext(ctx, "ECS scheduler: lease config invalid", "holder", sc.holder, "err", sc.leaseErr)
+		return
+	}
 	leaseDone := make(chan struct{})
 	go func() {
 		defer close(leaseDone)
-		// The leader owns the Layer-2 bus subscriptions and the instance-role
-		// converge pass. A node that can do neither stands down rather than hold
-		// the lease and process nothing.
-		kvsync.RunElection(ctx, sc.lease, sc.onGained, sc.unsubscribeBus)
+		sc.lease.Run(ctx)
 	}()
 
 	// Two loops rather than four tickers, split by what wakes them. Services are

spinifex/domains/ecs/scheduler_test.go

@@ -6,9 +6,7 @@ package ecs
 
 import (
 	"context"
-	"sync"
 	"testing"
-	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ecs"
@@ -16,67 +14,26 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
-	"github.com/nats-io/nats.go"
 )
 
-// testScheduler is holder's scheduler on a sync bucket of its own, as a
-// separate daemon's would be.
-func testScheduler(t *testing.T, nc *nats.Conn, svc *Service, holder string) *Scheduler {
-	t.Helper()
-	return NewScheduler(nc, svc, NewSchedulerLease(kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc)), holder))
-}
-
-// electOnce runs sc's election with the scheduler's own callbacks and returns
-// what its first onGained returned. stop ends the election and waits for it.
-func electOnce(t *testing.T, sc *Scheduler) (stop func(), gainedErr error) {
-	t.Helper()
-	ctx, cancel := context.WithCancel(t.Context())
-	gained := make(chan error, 1)
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		kvsync.RunElection(ctx, sc.lease, func(ctx context.Context) error {
-			err := sc.onGained(ctx)
-			gained <- err
-			return err
-		}, sc.unsubscribeBus)
-	}()
-	stop = func() { cancel(); <-done }
-	t.Cleanup(stop)
-	select {
-	case gainedErr = <-gained:
-	case <-time.After(10 * time.Second):
-		t.Fatal("the election never gained the lease")
-	}
-	return sync.OnceFunc(stop), gainedErr
-}
-
-// requireLeaseKeyAbsent fails unless no node holds the scheduler's key.
-func requireLeaseKeyAbsent(t *testing.T, nc *nats.Conn, msg string) {
-	t.Helper()
-	_, _, err := kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc)).Load(t.Context(), KeyScheduler)
-	require.ErrorIs(t, err, kvsync.ErrNotFound, msg)
-}
-
 // TestScheduler_StandsDownWithoutIAM is the fleet outage this guards: a leader
 // that cannot converge instance roles holds the lease, writes no sts:AssumeRole
 // grant, and every agent's credentials lapse an hour later.
 func TestScheduler_StandsDownWithoutIAM(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, nc, js := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "")
-	sc := testScheduler(t, nc, svc, "holder-a")
+	sc := NewScheduler(nc, svc, "holder-a")
+	require.NoError(t, sc.leaseErr)
 
-	stop, err := electOnce(t, sc)
-	require.Error(t, err, "a node without IAM must refuse the lease")
-	stop()
+	assert.False(t, sc.lease.TryAcquire(t.Context()), "a node without IAM must refuse the lease")
 	assert.False(t, sc.lease.Held())
 	assert.Zero(t, sc.subCount())
 
 	// The key must be free, or a healthy node waits out the full TTL.
-	requireLeaseKeyAbsent(t, nc, "a node that stood down left its key")
+	kv, err := InitLeaderBucket(t.Context(), js)
+	require.NoError(t, err)
+	_, err = kv.Get(t.Context(), schedulerLeaderKey)
+	require.Error(t, err)
 }
 
 // TestConvergeInstanceRoles_RewritesProvisionedAccounts pins the pass that
@@ -92,7 +49,7 @@ func TestConvergeInstanceRoles_RewritesProvisionedAccounts(t *testing.T) {
 	require.NoError(t, err)
 
 	iam.RolePolicies[ecsInstanceRoleName] = `{"Version":"2012-10-17","Statement":[]}`
-	sc := testScheduler(t, nc, svc, "holder-a")
+	sc := NewScheduler(nc, svc, "holder-a")
 	require.NoError(t, sc.convergeInstanceRoles(t.Context()))
 
 	assert.Contains(t, iam.RolePolicies[ecsInstanceRoleName], "sts:AssumeRole")
@@ -108,7 +65,7 @@ func TestConvergeInstanceRoles_SkipsAccountsWithoutTheRole(t *testing.T) {
 	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
 	require.NoError(t, err)
 
-	sc := testScheduler(t, nc, svc, "holder-a")
+	sc := NewScheduler(nc, svc, "holder-a")
 	require.NoError(t, sc.convergeInstanceRoles(t.Context()))
 
 	assert.Zero(t, iam.CreateRoleCalls)
@@ -120,7 +77,7 @@ func TestConvergeInstanceRoles_SkipsAccountsWithoutTheRole(t *testing.T) {
 // caller that reaches the pass without going through leader election.
 func TestConvergeInstanceRoles_NilIAMIsAnError(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
-	sc := testScheduler(t, nc, NewService(nc, testRegion, ""), "holder-a")
+	sc := NewScheduler(nc, NewService(nc, testRegion, ""), "holder-a")
 
 	err := sc.convergeInstanceRoles(t.Context())
 	require.Error(t, err)
@@ -137,59 +94,3 @@ func TestECSInstanceRolePolicyDoc_GrantsAssumeRole(t *testing.T) {
 	assert.Contains(t, doc, `"ecs:*"`)
 	assert.Contains(t, doc, `"ecr:GetAuthorizationToken"`)
 }
-
-// containerInstanceStatus is i-1's status in the web cluster.
-func containerInstanceStatus(t *testing.T, svc *Service) string {
-	t.Helper()
-	di, err := svc.DescribeContainerInstances(context.Background(), &ecs.DescribeContainerInstancesInput{
-		Cluster: aws.String("web"), ContainerInstances: []*string{aws.String("i-1")},
-	}, testAccountID)
-	require.NoError(t, err)
-	return aws.StringValue(di.ContainerInstances[0].Status)
-}
-
-// A leader that loses the lease must drop the bus and stop reaping, or two
-// nodes would mark instances DRAINING and stop their tasks at once. Re-elected,
-// it takes both back up.
-func TestScheduler_LosingTheLeaseDropsTheBusAndTheReaperUntilReElection(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
-	require.NoError(t, err)
-	registerInstance(t, svc, "web", "i-1", 1024, 2048)
-	kv, err := svc.bucket(t.Context(), testAccountID)
-	require.NoError(t, err)
-	var rec InstanceRecord
-	_, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
-	require.NoError(t, err)
-	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
-	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
-
-	store := kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc))
-	sc := NewScheduler(nc, svc, kvsync.NewLease(store, KeyScheduler, "holder-a", 300*time.Millisecond))
-	ctx, cancel := context.WithCancel(t.Context())
-	done := make(chan struct{})
-	go func() { defer close(done); kvsync.RunElection(ctx, sc.lease, sc.onGained, sc.unsubscribeBus) }()
-	t.Cleanup(func() { cancel(); <-done })
-
-	held := func() bool { return sc.lease.Held() && sc.subCount() == 4 }
-	require.Eventually(t, held, 5*time.Second, 10*time.Millisecond, "the leader subscribes to the four bus subjects")
-
-	syncKV, err := store.KV(t.Context())
-	require.NoError(t, err)
-	_, err = syncKV.Put(t.Context(), KeyScheduler, []byte(`{"holder":"holder-b","generation":0}`))
-	require.NoError(t, err)
-	require.Eventually(t, func() bool { return !sc.lease.Held() && sc.subCount() == 0 }, 5*time.Second, 10*time.Millisecond,
-		"a lost lease must drop every bus subscription")
-
-	revisit, err := sc.timerPass(t.Context())
-	require.NoError(t, err)
-	assert.Equal(t, leaseRefresh, revisit)
-	assert.Equal(t, "ACTIVE", containerInstanceStatus(t, svc), "a node that lost the lease must not reap")
-
-	require.NoError(t, syncKV.Delete(t.Context(), KeyScheduler))
-	require.Eventually(t, held, 5*time.Second, 10*time.Millisecond, "re-elected, the bus comes back")
-	_, err = sc.timerPass(t.Context())
-	require.NoError(t, err)
-	assert.Equal(t, "DRAINING", containerInstanceStatus(t, svc), "the re-elected leader reaps again")
-}

spinifex/domains/ecs/service_test.go

@@ -14,7 +14,6 @@ import (
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -572,7 +571,7 @@ func TestScheduler_ReapBucket_StopsStaleInstanceTasks(t *testing.T) {
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
-	sc := testScheduler(t, nc, svc, "test-holder")
+	sc := NewScheduler(nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	di, err := svc.DescribeContainerInstances(context.Background(), &ecs.DescribeContainerInstancesInput{
@@ -587,38 +586,32 @@ func TestScheduler_ReapBucket_StopsStaleInstanceTasks(t *testing.T) {
 	assert.Equal(t, stoppedReasonReaped, aws.StringValue(dt.Tasks[0].StoppedReason))
 }
 
-// Leader election: the first holder wins; a second node, and a second attempt
-// in the same process, are both refused while it leads.
+// Leader election: first holder wins Create; a second holder is rejected.
 func TestScheduler_AcquireLease_SingleLeader(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	a := testScheduler(t, nc, svc, "holder-a")
-	b := testScheduler(t, nc, svc, "holder-b")
-	tok, err := a.lease.TryLock(t.Context())
-	require.NoError(t, err)
-	t.Cleanup(func() { a.lease.Unlock(context.Background(), tok) })
-
-	_, err = b.lease.TryLock(t.Context())
-	assert.ErrorAs(t, err, new(*kvsync.HeldError))
-	_, err = a.lease.TryLock(t.Context())
-	assert.ErrorAs(t, err, new(*kvsync.HeldError), "a second attempt in the leading process is refused")
-	assert.True(t, a.lease.Held(), "a refused second attempt keeps leadership")
+	a := NewScheduler(nc, svc, "holder-a")
+	b := NewScheduler(nc, svc, "holder-b")
+	require.True(t, a.lease.TryAcquire(t.Context()))
+	assert.False(t, b.lease.TryAcquire(t.Context()))
+	assert.True(t, a.lease.TryAcquire(t.Context()), "a holder re-attempting keeps leadership")
 }
 
-// Leadership owns the Layer-2 bus subscriptions. This pins the wiring: gaining
-// the lease subscribes, losing it unsubscribes.
+// Leadership owns the Layer-2 bus subscriptions. This pins the wiring — gaining
+// the lease subscribes, losing it unsubscribes — which is the regression this
+// conversion could introduce and the one thing kvlease cannot check itself.
 func TestScheduler_LeadershipWiresBusSubscriptions(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	sc := testScheduler(t, nc, svc, "holder-a")
+	sc := NewScheduler(nc, svc, "holder-a")
+	require.NoError(t, sc.leaseErr)
 
 	require.Zero(t, sc.subCount(), "subscriptions must not exist before election")
 
-	stop, err := electOnce(t, sc)
-	require.NoError(t, err)
+	require.True(t, sc.lease.TryAcquire(t.Context()))
 	assert.Equal(t, 4, sc.subCount(), "leader must own the register, heartbeat, task-state and reconcile-wake subscriptions")
 
-	stop()
+	sc.lease.Release(t.Context())
 	assert.Zero(t, sc.subCount(), "a released leader must drop its subscriptions")
 }
 
@@ -627,21 +620,20 @@ func TestScheduler_LeadershipWiresBusSubscriptions(t *testing.T) {
 func TestScheduler_LoserHoldsNoSubscriptions(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	a := testScheduler(t, nc, svc, "holder-a")
-	b := testScheduler(t, nc, svc, "holder-b")
+	a := NewScheduler(nc, svc, "holder-a")
+	b := NewScheduler(nc, svc, "holder-b")
 
-	_, err := electOnce(t, a)
-	require.NoError(t, err)
-	_, err = b.lease.TryLock(t.Context())
-	require.ErrorAs(t, err, new(*kvsync.HeldError))
+	require.True(t, a.lease.TryAcquire(t.Context()))
+	require.False(t, b.lease.TryAcquire(t.Context()))
 	assert.Zero(t, b.subCount())
 	assert.Equal(t, 4, a.subCount())
 }
 
 func TestScheduler_RunReleasesLeaseOnShutdown(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, nc, js := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	sc := testScheduler(t, nc, svc, "holder-a")
+	sc := NewScheduler(nc, svc, "holder-a")
+	require.NoError(t, sc.leaseErr)
 
 	ctx, cancel := context.WithCancel(t.Context())
 	done := make(chan struct{})
@@ -651,7 +643,10 @@ func TestScheduler_RunReleasesLeaseOnShutdown(t *testing.T) {
 	cancel()
 	<-done
 
-	requireLeaseKeyAbsent(t, nc, "Run returned with the lease key still present")
+	kv, err := InitLeaderBucket(t.Context(), js)
+	require.NoError(t, err)
+	_, err = kv.Get(t.Context(), schedulerLeaderKey)
+	require.Error(t, err, "Run returned with the lease key still present")
 }
 
 // --- GPU placement dimension (Epic C2) ---

spinifex/domains/ecs/store.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -17,11 +19,18 @@ import (
 // cluster, task-definition, task, service, and container-instance state. It is
 // created lazily on first cluster touch (no daemon-boot pre-creation) so accounts
 // without any ECS clusters never grow a bucket (ecs-v1.md Q2).
-
+//
+// Shared leader bucket "spinifex-ecs-leader" holds 60s-TTL CAS locks keyed by
+// "{accountID}/{clusterName}" for the per-cluster scheduler leader election
+// (Q3). Created once at daemon boot.
 const (
 	KVBucketECSAccountPrefix  = "ecs-account-"
 	KVBucketECSAccountVersion = 1
 	KVBucketECSAccountHistory = 1
+
+	KVBucketECSLeader        = "spinifex-ecs-leader"
+	KVBucketECSLeaderVersion = 1
+	KVBucketECSLeaderTTL     = 60 * time.Second
 )
 
 // Key-path helpers for the per-account bucket. The layout matches ecs-v1.md Q2.
@@ -110,15 +119,15 @@ func CapacityProviderKey(name string) string {
 	return CapacityProvidersPrefix() + name
 }
 
-// Store is the per-daemon ECS KV handle. Per-account buckets are accessed via
-// the package-level factories below.
+// Store is the per-daemon ECS KV handle. Per-account and leader buckets are
+// accessed via the package-level factories below.
 type Store struct {
 	nc *nats.Conn
 }
 
 // NewStore constructs a Store bound to the supplied NATS connection. It does not
 // touch JetStream — per-account buckets are created lazily by
-// GetOrCreateAccountBucket.
+// GetOrCreateAccountBucket and the leader bucket by InitLeaderBucket.
 func NewStore(nc *nats.Conn) (*Store, error) {
 	if nc == nil {
 		return nil, errors.New("ecs store: nats connection is nil")
@@ -149,7 +158,7 @@ func accountBuckets(ctx context.Context, nc *nats.Conn) ([]accountBucket, error)
 	if err != nil {
 		return nil, fmt.Errorf("jetstream: %w", err)
 	}
-	names, err := kvstore.BucketNames(ctx, js)
+	names, err := kvutil.BucketNames(ctx, js)
 	if err != nil {
 		return nil, err
 	}
@@ -167,7 +176,7 @@ func accountBuckets(ctx context.Context, nc *nats.Conn) ([]accountBucket, error)
 // return the existing handle.
 func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) {
 	bucket := AccountBucketName(accountID)
-	kv, err := kvstore.GetOrCreateBucket(ctx, js, bucket, KVBucketECSAccountHistory)
+	kv, err := kvutil.GetOrCreateBucket(ctx, js, bucket, KVBucketECSAccountHistory)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create ECS per-account KV bucket %s: %w", bucket, err)
 	}
@@ -176,3 +185,13 @@ func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accou
 	}
 	return kv, nil
 }
+
+// InitLeaderBucket creates (or attaches to) the shared spinifex-ecs-leader bucket
+// used for per-cluster scheduler leader-lease CAS locks.
+func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {
+	return kvlease.OpenBucket(ctx, js, kvlease.BucketConfig{
+		Name:    KVBucketECSLeader,
+		TTL:     KVBucketECSLeaderTTL,
+		Version: KVBucketECSLeaderVersion,
+	})
+}

spinifex/domains/ecs/store_test.go

@@ -1,9 +1,13 @@
 package ecs
 
 import (
+	"encoding/json"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -42,6 +46,71 @@ func TestGetOrCreateAccountBucket_Idempotent(t *testing.T) {
 	assert.Equal(t, kv1.Bucket(), kv2.Bucket())
 }
 
+func TestInitLeaderBucket_Idempotent(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	js := testutil.NewJetStream(t, nc)
+
+	kv1, err := InitLeaderBucket(t.Context(), js)
+	require.NoError(t, err)
+	require.NotNil(t, kv1)
+	assert.Equal(t, KVBucketECSLeader, kv1.Bucket())
+
+	kv2, err := InitLeaderBucket(t.Context(), js)
+	require.NoError(t, err)
+	require.NotNil(t, kv2)
+	assert.Equal(t, KVBucketECSLeader, kv2.Bucket())
+}
+
+// The scheduler calls InitLeaderBucket on every tick. Creating first and falling
+// back on already-exists reaches the same handle, so idempotency alone cannot
+// tell the two orders apart — only the API traffic can.
+//
+// On a multi-node cluster the stored bucket is replicated while this call asks
+// for the default, and JetStream answers the mismatch with an error rather than
+// a no-op: 10058, once per tick, forever. That is invisible from the caller and
+// showed up only as a 16% JetStream API error rate on the meta leader.
+func TestInitLeaderBucket_AttachesWithoutCreating(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+
+	// Watch from before the first call. The server queues the API advisory after
+	// the reply the caller waits on, so subscribing once the bucket exists can
+	// still catch the create that first call legitimately made.
+	creates := make(chan string, 8)
+	sub, err := nc.Subscribe("$JS.EVENT.ADVISORY.API", func(m *nats.Msg) {
+		var adv struct {
+			Subject string `json:"subject"`
+		}
+		if json.Unmarshal(m.Data, &adv) == nil &&
+			strings.HasPrefix(adv.Subject, "$JS.API.STREAM.CREATE.") {
+			creates <- adv.Subject
+		}
+	})
+	require.NoError(t, err)
+	defer func() { _ = sub.Unsubscribe() }()
+	require.NoError(t, nc.Flush())
+
+	_, err = InitLeaderBucket(t.Context(), js)
+	require.NoError(t, err)
+
+	// Take the first, legitimate create off the channel before the call under
+	// test, so anything left there afterwards can only have come from it.
+	select {
+	case <-creates:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the first InitLeaderBucket must create the bucket")
+	}
+
+	_, err = InitLeaderBucket(t.Context(), js)
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
+
+	select {
+	case subject := <-creates:
+		t.Fatalf("InitLeaderBucket issued %s on an existing bucket; attach before create", subject)
+	case <-time.After(250 * time.Millisecond):
+	}
+}
+
 // Key-path helpers must produce the ecs-v1.md Q2 layout exactly: prefixes are
 // what the List* enumerations watch, so a drift here silently breaks listing.
 func TestKeyPaths(t *testing.T) {