refactor: fold kvutil into kvstore and rename Claim to Take (#1608)

* refactor: fold kvutil into kvstore and rename Claim to Take

kvutil was the layer under kvstore, so its bucket, replica, version, key
and CAS helpers now live in kvstore and every importer changes only its
import path and qualifier. kvutil.Claim becomes kvstore.Take so that it
does not read as a lock claim, and the kvutil package is deleted.

The bucket-creation invariant now exempts kvstore/buckets.go, the one
file holding the chokepoint, rather than the whole package.

* refactor: merge the kvstore test exports into one file

* refactor: move the kvutil callers added on dev to kvstore

.github/workflows/e2e-nightly.yml

@@ -578,6 +578,8 @@ jobs:
              echo \"::group::suite ${SUITE}\"; \
              date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
              GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               SPINIFEX_NODES='${{ steps.vm.outputs.wan_ip }}' SPINIFEX_SSH_USER=tf-user \
+               SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                ARTIFACT_DIR='${ARTIFACT_DIR}' \
                ./${SUITE}.test -test.v -test.timeout=${TMO} \
                2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log'; \

build/microvm/init.sh

@@ -32,6 +32,8 @@ load_mod qemu_fw_cfg || load_mod fw_cfg_sysfs || true
 load_mod failover || true
 load_mod net_failover || true
 load_mod virtio_net || true
+# lb-agent probes each data NIC's gateway on an AF_PACKET socket.
+load_mod af_packet || true
 
 NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
 i=0

build/microvm/lambda-runtime/init.sh

@@ -16,8 +16,8 @@ load_mod() {
     done
     return 1
 }
-# lambda-agent reads its identity from fw_cfg and unloads qemu_fw_cfg before
-# anything else runs. virtio_net needs net_failover and failover, and the
+# lambda-agent reads its identity from fw_cfg, or the configuration disk, and
+# seals it before anything else runs. virtio_net needs net_failover and failover, and the
 # insmod fallback resolves no dependencies, so load the chain bottom-up.
 load_mod qemu_fw_cfg || echo "[init] qemu_fw_cfg did not load" >/dev/console
 load_mod failover || true
@@ -26,8 +26,30 @@ load_mod virtio_net || echo "[init] virtio_net did not load" >/dev/console
 
 ip link set lo up
 
+# Without fw_cfg data the executor attaches the same entries as a tar archive
+# on the second disk; lambda-agent reads them here and erases the disk before
+# the function starts. qemu_fw_cfg loads without a device, so test the entry.
+CFG_ROOT=/spinifex-cfg
+FWCFG_ROOT=/sys/firmware/qemu_fw_cfg
+if [ ! -f "$FWCFG_ROOT/by_name/opt/spinifex/netcfg/raw" ]; then
+    load_mod virtio_blk || echo "[init] virtio_blk did not load" >/dev/console
+    i=0
+    while [ "$i" -lt 25 ] && [ ! -b /dev/vdb ]; do
+        sleep 0.04
+        i=$((i + 1))
+    done
+    if [ -b /dev/vdb ]; then
+        mkdir -m 0700 "$CFG_ROOT"
+        if ! mount -t tmpfs -o mode=0700,size=1m tmpfs "$CFG_ROOT" ||
+            ! tar -xf /dev/vdb -C "$CFG_ROOT"; then
+            echo "[init] configuration disk not readable" >/dev/console
+        fi
+        FWCFG_ROOT=$CFG_ROOT
+    fi
+fi
+
 # The guest's addresses come from the same netcfg entry as other system VMs.
-NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
+NETCFG=$FWCFG_ROOT/by_name/opt/spinifex/netcfg/raw
 i=0
 while [ "$i" -lt 25 ] && [ ! -f "$NETCFG" ]; do
     sleep 0.04

cmd/lambda-agent/connect.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"os"
 
 	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
@@ -25,9 +26,22 @@ type startup struct {
 }
 
 func defaultStartup() startup {
+	if _, err := os.Stat(hostlink.ConfigDiskRoot); err == nil {
+		return configDiskStartup(hostlink.ConfigDiskRoot, hostlink.ConfigDiskDevice)
+	}
 	return startup{fwCfgDir: hostlink.FwCfgDir, fwCfgRoot: hostlink.FwCfgRoot, unload: hostlink.UnloadFwCfg}
 }
 
+// configDiskStartup reads the entries init extracted from the configuration
+// disk, and seals by erasing the disk and removing the extracted tree.
+func configDiskStartup(root, device string) startup {
+	return startup{
+		fwCfgDir:  root + "/by_name/" + lambdav1.FwCfgPrefix,
+		fwCfgRoot: root,
+		unload:    func() error { return hostlink.SealConfigDisk(device, root) },
+	}
+}
+
 // connect loads the identity, seals fw_cfg whether or not the load succeeded,
 // then dials the host. Under TLS 1.3 a host refusal is seen only on first read.
 func (s startup) connect(ctx context.Context) (*tls.Conn, lambdav1.Scope, int, error) {

cmd/lambda-agent/connect_configdisk_test.go

@@ -0,0 +1,43 @@
+//test:in-package — package main cannot be imported by an external test package.
+
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// An identity that does not load still seals the configuration disk: the
+// guest key on it is erased before the agent refuses to start.
+func TestConfigDiskStartup_RefusedLoadErasesDisk(t *testing.T) {
+	dir := t.TempDir()
+	root := filepath.Join(dir, "cfg")
+	require.NoError(t, os.MkdirAll(filepath.Join(root, "by_name"), 0o700))
+	device := filepath.Join(dir, "vdb")
+	require.NoError(t, os.WriteFile(device, bytes.Repeat([]byte("guest-key"), 512), 0o600))
+
+	_, _, code, err := configDiskStartup(root, device).connect(t.Context())
+	require.Error(t, err)
+	require.Equal(t, exitRefused, code)
+	got, rerr := os.ReadFile(device)
+	require.NoError(t, rerr)
+	require.Len(t, got, 9*512)
+	require.Equal(t, make([]byte, len(got)), got, "configuration disk erased")
+}
+
+// A disk that cannot be erased fails closed: the agent refuses, and the
+// extracted tree is kept so the seal check sees it.
+func TestConfigDiskStartup_UnerasableDiskRefuses(t *testing.T) {
+	dir := t.TempDir()
+	root := filepath.Join(dir, "cfg")
+	require.NoError(t, os.MkdirAll(filepath.Join(root, "by_name"), 0o700))
+
+	_, _, code, err := configDiskStartup(root, filepath.Join(dir, "missing")).connect(t.Context())
+	require.ErrorContains(t, err, "erase configuration disk")
+	require.Equal(t, exitRefused, code)
+	require.DirExists(t, root)
+}

cmd/lambda-agent/sdk_config_testlink.go

@@ -0,0 +1,65 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"path/filepath"
+)
+
+const (
+	sdkEndpointEntry = "test-sdk-endpoint"
+	sdkCAEntry       = "test-sdk-ca"
+	sdkBundlePath    = "/run/lambda-sdk/ca-bundle.pem"
+	systemBundlePath = "/etc/ssl/certs/ca-certificates.crt"
+)
+
+type sdkConfig struct {
+	endpoint string
+	ca       []byte
+}
+
+// readSDKConfig reads optional public settings before the boot config is sealed.
+func readSDKConfig(dir string) (*sdkConfig, error) {
+	endpoint, endpointErr := os.ReadFile(filepath.Join(dir, sdkEndpointEntry, "raw"))
+	ca, caErr := os.ReadFile(filepath.Join(dir, sdkCAEntry, "raw"))
+	if errors.Is(endpointErr, os.ErrNotExist) && errors.Is(caErr, os.ErrNotExist) {
+		return nil, nil
+	}
+	if endpointErr != nil || caErr != nil {
+		return nil, errors.New("incomplete SDK gateway settings")
+	}
+	u, err := url.Parse(string(endpoint))
+	if err != nil || u.Scheme != "https" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" ||
+		net.ParseIP(u.Hostname()) == nil || u.Port() == "" {
+		return nil, errors.New("invalid SDK gateway endpoint")
+	}
+	if len(ca) > 64<<10 || !x509.NewCertPool().AppendCertsFromPEM(ca) {
+		return nil, errors.New("invalid SDK gateway CA")
+	}
+	return &sdkConfig{endpoint: string(endpoint), ca: ca}, nil
+}
+
+// writeBundle combines the system roots with the cluster's public root.
+func (c *sdkConfig) writeBundle(path, systemPath string) error {
+	system, err := os.ReadFile(systemPath)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("read system CA bundle: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("create SDK CA directory: %w", err)
+	}
+	if err := os.Chmod(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("make SDK CA directory readable: %w", err)
+	}
+	bundle := append(append(system, '\n'), c.ca...)
+	if err := os.WriteFile(path, bundle, 0o644); err != nil {
+		return fmt.Errorf("write SDK CA bundle: %w", err)
+	}
+	return os.Chmod(path, 0o644)
+}

cmd/lambda-agent/sdk_config_testlink_test.go

@@ -0,0 +1,106 @@
+//go:build lambdatestlink
+
+//test:in-package exercises the unexported boot configuration reader
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSDKConfig_OptionalAndInvalid(t *testing.T) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+	defer server.Close()
+	ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
+	for _, tc := range []struct {
+		name     string
+		endpoint []byte
+		ca       []byte
+		wantErr  bool
+		want     bool
+	}{
+		{name: "absent"},
+		{name: "valid", endpoint: []byte(server.URL), ca: ca, want: true},
+		{name: "missing ca", endpoint: []byte(server.URL), wantErr: true},
+		{name: "missing endpoint", ca: ca, wantErr: true},
+		{name: "invalid endpoint", endpoint: []byte("http://127.0.0.1:1234"), ca: ca, wantErr: true},
+		{name: "invalid ca", endpoint: []byte(server.URL), ca: []byte("not a certificate"), wantErr: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			for name, value := range map[string][]byte{sdkEndpointEntry: tc.endpoint, sdkCAEntry: tc.ca} {
+				if value != nil {
+					require.NoError(t, os.MkdirAll(filepath.Join(dir, name), 0o700))
+					require.NoError(t, os.WriteFile(filepath.Join(dir, name, "raw"), value, 0o600))
+				}
+			}
+			cfg, err := readSDKConfig(dir)
+			if tc.wantErr {
+				require.Error(t, err)
+				require.Nil(t, cfg)
+				return
+			}
+			require.NoError(t, err)
+			if !tc.want {
+				require.Nil(t, cfg)
+				return
+			}
+			bundle := filepath.Join(t.TempDir(), "sdk", "ca.pem")
+			require.NoError(t, cfg.writeBundle(bundle, filepath.Join(t.TempDir(), "absent")))
+			data, err := os.ReadFile(bundle)
+			require.NoError(t, err)
+			roots := x509.NewCertPool()
+			require.True(t, roots.AppendCertsFromPEM(data))
+			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
+			resp, err := client.Get(cfg.endpoint)
+			require.NoError(t, err)
+			defer resp.Body.Close()
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+		})
+	}
+}
+
+func TestSDKConfig_BundleKeepsSystemRoots(t *testing.T) {
+	cluster := httptest.NewTLSServer(nil)
+	defer cluster.Close()
+	clusterCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cluster.Certificate().Raw})
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	template := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test system root"},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+	require.NoError(t, err)
+	systemCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	dir := t.TempDir()
+	systemPath, bundlePath := filepath.Join(dir, "system.pem"), filepath.Join(dir, "sdk", "ca.pem")
+	require.NoError(t, os.WriteFile(systemPath, systemCA, 0o600))
+	require.NoError(t, (&sdkConfig{ca: clusterCA}).writeBundle(bundlePath, systemPath))
+	bundle, err := os.ReadFile(bundlePath)
+	require.NoError(t, err)
+	roots := x509.NewCertPool()
+	require.True(t, roots.AppendCertsFromPEM(bundle))
+	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
+	resp, err := client.Get(cluster.URL)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	systemCert, err := x509.ParseCertificate(der)
+	require.NoError(t, err)
+	_, err = systemCert.Verify(x509.VerifyOptions{Roots: roots})
+	require.NoError(t, err)
+}

cmd/lambda-agent/start_testlink.go

@@ -12,11 +12,14 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"sort"
+	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/lambdaenv"
 )
 
 const (
@@ -31,13 +34,17 @@ type testlink struct {
 	pkg            taskPackage
 	runtimeAddr    string
 	startBootstrap func(ctx context.Context, env []string) (*exec.Cmd, error)
+	caBundlePath   string
+	systemCAPath   string
 }
 
 func start(ctx context.Context) int {
 	t := testlink{
-		startup:     defaultStartup(),
-		pkg:         devTaskPackage(),
-		runtimeAddr: runtimeAddr,
+		startup:      defaultStartup(),
+		pkg:          devTaskPackage(),
+		runtimeAddr:  runtimeAddr,
+		caBundlePath: sdkBundlePath,
+		systemCAPath: systemBundlePath,
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			return agent.StartBootstrap(ctx, taskRoot+"/bootstrap", env, os.Stdout, os.Stderr)
 		},
@@ -53,6 +60,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 	// The package values are fw_cfg entries, so read them before connect seals it.
 	spec, specErr := readPackageSpec(t.fwCfgDir)
 	grant, grantErr := readGrant(t.fwCfgDir)
+	sdk, sdkErr := readSDKConfig(t.fwCfgDir)
 	conn, scope, code, err := t.connect(parent)
 	if err != nil {
 		return code, errors.Join(err, specErr, grantErr)
@@ -85,6 +93,18 @@ func (t testlink) run(parent context.Context) (int, error) {
 	}
 	cred := *first.Credential
 	first.Credential = nil
+	if err := lambdaenv.Validate(cred.Environment); err != nil {
+		return exitRefused, errors.New("first frame carries an invalid function environment")
+	}
+	if sdkErr != nil {
+		slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", sdkErr)
+	}
+	if sdk != nil {
+		if err := sdk.writeBundle(t.caBundlePath, t.systemCAPath); err != nil {
+			slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", err)
+			sdk = nil
+		}
+	}
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -100,7 +120,8 @@ func (t testlink) run(parent context.Context) (int, error) {
 
 	ctx, cancel := context.WithCancelCause(parent)
 	defer cancel(nil)
-	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred))
+	region, _, _ := strings.Cut(string(scope), "/")
+	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred, region, sdk, t.caBundlePath))
 	if err != nil {
 		return exitLink, fmt.Errorf("start bootstrap: %w", err)
 	}
@@ -124,18 +145,35 @@ func (t testlink) run(parent context.Context) (int, error) {
 	return exitClosed, nil
 }
 
-// bootstrapEnv is the bootstrap's whole environment: the runtime variables and
-// the execution role's three credential variables.
-func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential) []string {
-	return []string{
-		"AWS_LAMBDA_RUNTIME_API=" + runtimeAPI,
-		"_HANDLER=" + handler,
-		"LAMBDA_TASK_ROOT=" + taskRoot,
-		"PATH=/bin:/sbin:/usr/bin",
-		"AWS_ACCESS_KEY_ID=" + cred.AccessKeyID,
-		"AWS_SECRET_ACCESS_KEY=" + cred.SecretAccessKey,
-		"AWS_SESSION_TOKEN=" + cred.SessionToken,
+// bootstrapEnv supplies the runtime, execution-role and SDK settings.
+func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential, region string, sdk *sdkConfig, caPath string) []string {
+	vars := map[string]string{
+		"AWS_LAMBDA_RUNTIME_API": runtimeAPI,
+		"_HANDLER":               handler,
+		"LAMBDA_TASK_ROOT":       taskRoot,
+		"PATH":                   "/bin:/sbin:/usr/bin",
+		"AWS_ACCESS_KEY_ID":      cred.AccessKeyID,
+		"AWS_SECRET_ACCESS_KEY":  cred.SecretAccessKey,
+		"AWS_SESSION_TOKEN":      cred.SessionToken,
+		"AWS_REGION":             region,
+		"AWS_DEFAULT_REGION":     region,
 	}
+	if sdk != nil {
+		vars["AWS_ENDPOINT_URL"], vars["AWS_CA_BUNDLE"] = sdk.endpoint, caPath
+	}
+	for k, v := range cred.Environment {
+		vars[k] = v
+	}
+	keys := make([]string, 0, len(vars))
+	for k := range vars {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	env := make([]string, 0, len(keys))
+	for _, k := range keys {
+		env = append(env, k+"="+vars[k])
+	}
+	return env
 }
 
 // shutdownGrace is how long past the invocation deadline the agent waits for

cmd/lambda-agent/start_testlink_test.go

@@ -7,8 +7,10 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/pem"
 	"io"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -70,6 +72,7 @@ type bootstrapStarts struct {
 	sealed  bool
 	// handlerOut receives the _HANDLER the bootstrap process itself saw.
 	handlerOut string
+	caBundle   string
 	// env is the environment the agent gave the bootstrap.
 	env []string
 }
@@ -94,6 +97,7 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 	b.started = make(chan struct{}, 1)
 	b.task = t.TempDir()
 	b.handlerOut = filepath.Join(t.TempDir(), "handler")
+	b.caBundle = filepath.Join(t.TempDir(), "sdk", "ca-bundle.pem")
 	return testlink{
 		startup: g.startup(),
 		pkg: taskPackage{
@@ -102,7 +106,9 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 			mountDir:   func(string) error { return nil },
 			sealDir:    func(string) error { b.sealed = true; return nil },
 		},
-		runtimeAddr: "127.0.0.1:0",
+		runtimeAddr:  "127.0.0.1:0",
+		caBundlePath: b.caBundle,
+		systemCAPath: filepath.Join(t.TempDir(), "system-ca.pem"),
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			b.n++
 			b.env = env
@@ -176,10 +182,87 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		"AWS_ACCESS_KEY_ID=" + testCredential.AccessKeyID,
 		"AWS_SECRET_ACCESS_KEY=" + testCredential.SecretAccessKey,
 		"AWS_SESSION_TOKEN=" + testCredential.SessionToken,
+		"AWS_REGION=ap-southeast-2",
+		"AWS_DEFAULT_REGION=ap-southeast-2",
 	}, b.env, "the bootstrap gets exactly the runtime and credential variables")
 	assert.NotEmpty(t, envValue(b.env, "AWS_LAMBDA_RUNTIME_API"))
 }
 
+func TestTestlinkProvidesSDKSettingsToBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+	defer server.Close()
+	h := newHost(t)
+	h.accept(ctx)
+	entries := h.entries()
+	entries[sdkEndpointEntry] = []byte(server.URL)
+	entries[sdkCAEntry] = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, entries))
+	conn := h.guest(t)
+	defer conn.Close()
+	hs := hostlink.NewTestStream(conn)
+	req := hostlink.Request{Credential: &testCredential, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
+	require.NoError(t, hs.HostSend(req))
+	_, err := hs.HostResult(ctx)
+	require.NoError(t, err)
+	res := wait(t, done, "agent to stop")
+	require.NoError(t, res.err)
+	assert.Equal(t, server.URL, envValue(b.env, "AWS_ENDPOINT_URL"))
+	assert.Equal(t, b.caBundle, envValue(b.env, "AWS_CA_BUNDLE"))
+	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_REGION"))
+	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_DEFAULT_REGION"))
+	ca, err := os.ReadFile(b.caBundle)
+	require.NoError(t, err)
+	assert.Contains(t, string(ca), string(entries[sdkCAEntry]))
+	info, err := os.Stat(b.caBundle)
+	require.NoError(t, err)
+	assert.Equal(t, os.FileMode(0o644), info.Mode().Perm())
+}
+
+func TestTestlinkProvidesFunctionEnvironment(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
+	conn := h.guest(t)
+	defer conn.Close()
+	hs := hostlink.NewTestStream(conn)
+	cred := testCredential
+	cred.Environment = map[string]string{"GREETING": "hello $WORLD", "PATH": "/my/bin"}
+	req := hostlink.Request{Credential: &cred, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
+	require.NoError(t, hs.HostSend(req))
+	_, err := hs.HostResult(ctx)
+	require.NoError(t, err)
+	require.NoError(t, wait(t, done, "agent to stop").err)
+	assert.Equal(t, "hello $WORLD", envValue(b.env, "GREETING"))
+	assert.Equal(t, "/my/bin", envValue(b.env, "PATH"))
+	assert.Equal(t, testCredential.SecretAccessKey, envValue(b.env, "AWS_SECRET_ACCESS_KEY"))
+}
+
+func TestTestlinkRejectsReservedFunctionEnvironment(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
+	conn := h.guest(t)
+	defer conn.Close()
+	cred := testCredential
+	cred.Environment = map[string]string{"AWS_SECRET_ACCESS_KEY": "forged"}
+	req := hostlink.Request{Credential: &cred, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
+	result := wait(t, done, "agent to refuse the environment")
+	assert.Equal(t, exitRefused, result.code)
+	assert.Error(t, result.err)
+	assert.NotContains(t, result.err.Error(), "forged")
+	assert.Zero(t, b.n)
+}
+
 var testCredential = hostlink.Credential{AccessKeyID: "ASIAGUESTKEY", SecretAccessKey: "guest-secret", SessionToken: "guest-token"}
 
 func envValue(env []string, name string) string {

contracts/lambda/v1/README.md

@@ -21,13 +21,16 @@ Tenant code may be able to reach the listener, but it cannot open the agent's co
 **Authentication:** mutual TLS 1.3, bound to the environment ID and realization generation, using material the user runtime cannot read (ADR-0009:S17).
 For each generation the executor creates ephemeral self-signed host and guest certificates whose single URI SAN is `spinifex-lambda://<Region>/<account>/<function ID>/<environment ID>/<generation>/<role>`, with role `host` or `guest`.
 Each end validates the peer certificate against the one pinned peer certificate and separately requires that URI SAN with the opposite role; hostname verification is not relied on.
-ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/2` on both host and guest, whose first frame carries the execution-role credential, so a test-link end and a release end fail the handshake; release builds are unchanged.
-In the same builds, the gateway reaches the host daemon on the local abstract socket `spinifex-lambda-testlink-control/3`, and each request carries the runtime-image digest the gateway was built with; the daemon refuses a missing or different digest before starting the attempt, and a gateway and daemon on different socket names never connect.
+ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/3` on both host and guest, whose first frame carries the execution-role credential and function environment, so a test-link end and a release end fail the handshake; release builds are unchanged.
+In the same builds, the gateway reaches the host daemon on the local abstract socket `spinifex-lambda-testlink-control/4`, and each request carries the runtime-image digest the gateway was built with; the daemon refuses a missing or different digest before starting the attempt, and a gateway and daemon on different socket names never connect.
 On that socket a request's raw payload is at most 6 MiB, checked by the gateway before encoding and by the daemon after decoding.
 Its encoded frame is at most the base64 length of 6 MiB plus 64 KiB of metadata headroom and 64 bytes of field slack, and a receiver checks the frame length before allocating; metadata has no separate bound within the frame.
-The credential travels as its own frame of at most 8 KiB, and the reply has the same payload and frame bounds as the request.
+The credential and environment travel as their own frame of at most 40 KiB, and the reply has the same payload and frame bounds as the request.
 The daemon holds at most two connections at once, from accept to close, and leaves the rest unaccepted in the listen backlog.
 Under TLS 1.3 the guest's handshake can complete before the host refuses its certificate, so the guest treats the connection as accepted only after the host's first message.
+In the same builds, the function owner moves a created function with stored code from `Pending` to `Active` once this node's daemon serves the pinned image and the cluster configuration has exactly one node; a code copy over the test-link bound stays `Pending`, and `StateReason` names the unmet condition.
+Release builds never leave `Pending`.
+The Mulga deploy role installs a test-link build with `deploy_lambda_testlink=true`; this profile does not meet Q-199 fail-closed revocation, and its guests take management addresses whose record can expire (mulga-0hsub).
 
 **Bootstrap material:** the executor passes these fw_cfg entries under `opt/spinifex/lambda/`, and the agent reads them as root and unloads `qemu_fw_cfg` before user code starts.
 
@@ -41,6 +44,8 @@ Under TLS 1.3 the guest's handshake can complete before the host refuses its cer
 
 The guest's own interface settings come from `opt/spinifex/netcfg`, the entry other system VMs use.
 The agent unloads `qemu_fw_cfg` even when an entry is missing, and refuses to start if the fw_cfg tree is still readable afterwards.
+On a VMM without fw_cfg (the test-link Firecracker executor), the executor attaches the same entries as a tar archive on the second, writable virtio-blk disk, laid out as `by_name/<entry>/raw`.
+The runtime image's init extracts it to a root-only tmpfs at `/spinifex-cfg`, and the agent reads the entries there, then overwrites the whole disk with zeros and removes the tree; it refuses to start if the tree is still present.
 Until the message format exists, the agent authenticates and holds the connection but accepts no host message and starts no user code.
 
 **Authority:** one environment's attempt only.

docs/testing/e2e-coverage.md

@@ -108,6 +108,8 @@ Three entry points, in the order they run:
 
 **Its own suite and its own cell, for instancerecovery's reasons plus one.** It takes a node away, so anything sharing the environment reports that outage as its own failure; it needs its own VPC, public subnet and a backend placement no shared fixture describes; and a failure here has to read as "the load balancer did not survive its host" rather than as one more red subtest in a suite about load balancer features.
 
+**A dead data NIC is the other half, and there the state does change.** `LBSelfHealsADeadDataNIC` disables one data NIC of a running load balancer guest through a second QMP client and runs for an internet-facing NLB and an internal ALB. The backends and an unrelated customer instance in the same subnet are launched more than 21 minutes before the fault, so a repair that reached a customer instance would find one the platform has long stopped expecting to change. A state watch samples `DescribeLoadBalancers` every two seconds and requires the sequence `active`, `active_impaired`, `active` with nothing in between, while the internet-facing run also watches the name continuously and serves traffic through it within five minutes. Afterwards the guest, its interfaces and its address are unchanged but its process is not, the backends and the bystander keep their instance ID, launch time and process (ADR-0003:S7), and the daemon journals exactly one `ELBv2 data-path repair started` line carrying the fields an operator needs. The internal ALB has no client inside the VPC, so it asserts the state sequence and target health rather than traffic through its name.
+
 ### `iam` — everything IAM and STS
 
 Policy evaluation against live requests rather than unit-level: a deny applying to the next request without a new credential, resource-scoped grants naming one key pair, a user rename needing `iam:UpdateUser` on both the old and the new name, `aws:SourceIp` seeing the real client address, assumed-role sessions carrying role policies, and revocation stopping one key alone.
@@ -129,7 +131,7 @@ The client reports on its serial console because the runner has no route into th
 
 ### `eks`, `ecs`, `ecr` — container services
 
-`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod whose spec names only a ServiceAccount with the role-arn annotation, where the pod-identity webhook injects the token volume, `AWS_*` environment, STS endpoint and CA bundle and the pod assumes the role, while a pod of an unannotated ServiceAccount is left untouched. The unmodified aws-load-balancer-controller chart 1.11.0 installs with the values an AWS user passes plus the gateway endpoint override, the Shield and WAF switches and the single-subnet feature gate, and reconciles an Ingress into an active ALB. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
+`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod whose spec names only a ServiceAccount with the role-arn annotation, where the pod-identity webhook injects the token volume, `AWS_*` environment, STS endpoint and CA bundle and the pod assumes the role, while a pod of an unannotated ServiceAccount is left untouched. The unmodified aws-load-balancer-controller chart 1.11.0 installs with the values an AWS user passes plus the gateway endpoint override, the Shield and WAF switches and the single-subnet feature gate, and reconciles an Ingress into an active ALB. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. The private-endpoint NIC of the cluster's endpoint NLB is disabled through QMP and every worker must be Ready again within ten minutes while the cluster stays `ACTIVE` and the NLB keeps its guest instance. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
 
 ### `rds` — managed databases
 

scripts/build-microvm-image.sh

@@ -209,6 +209,7 @@ echo "[build-microvm-image] nginx: binary + stream module present"
 #   virtio*.ko*    — virtio bus, net, blk, rng, console, and helpers
 #   *fw_cfg*.ko*   — QEMU fw_cfg sysfs driver (delivers boot blobs)
 #   *failover*.ko* — net_failover + failover (virtio_net deps for live migration)
+#   af_packet.ko*  — AF_PACKET sockets (lb-agent's per-NIC gateway ARP probe)
 # All other loadable modules are removed. Drivers compiled into the kernel
 # (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
 echo "[build-microvm-image] stripping kernel modules to virtio+fw_cfg only..."
@@ -222,6 +223,7 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
     find "$kernel_dir" -name "virtio*.ko.gz" -o -name "virtio*.ko" \
                        -o -name "*fw_cfg*.ko.gz" -o -name "*fw_cfg*.ko" \
                        -o -name "*failover*.ko.gz" -o -name "*failover*.ko" \
+                       -o -name "af_packet.ko.gz" -o -name "af_packet.ko" \
         2>/dev/null | while read -r mod; do
         rel="${mod#"$kernel_dir/"}"
         dest_dir="$tmp_keep/$(dirname "$rel")"
@@ -240,6 +242,15 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
     done)
     rm -rf "$tmp_keep"
 
+    # lb-agent's data-path probe needs AF_PACKET; without it every LB would
+    # report a dead NIC. Accept a kernel with CONFIG_PACKET built in.
+    if ! find "$kernel_dir" -name "af_packet.ko*" | grep -q .; then
+        if ! grep -q "CONFIG_PACKET=y" "$KERNEL_CONFIG" 2>/dev/null; then
+            echo "ERROR: af_packet.ko missing from the kept modules — lb-agent's data-path probe cannot open its socket" >&2
+            exit 1
+        fi
+    fi
+
     # Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
     # kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
     # produces an EMPTY modules.dep from compressed modules — the guest's

spinifex/agents/elbv2/agent.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
@@ -94,6 +95,18 @@ type Agent struct {
 	statsFn func(socketPath string) ([]ServerStatus, error)
 	// For testing: override the active health prober (nginx/NLB).
 	probeFn func(targets []HealthTarget) []ServerStatus
+
+	// Data-path probe: netcfg and sysfs locations, the ARP exchange, the
+	// instance ID lookup and per-NIC miss counters (overridable in tests).
+	netcfgPath   string
+	sysNetDir    string
+	probeWait    time.Duration
+	arpProbeFn   arpProbeFn
+	instanceIDFn func() (string, error)
+	instanceID   string
+	idSkip       int
+	probeMu      sync.Mutex
+	probeStates  map[string]*nicProbeState
 }
 
 // New creates a new LB agent for the given load balancer.
@@ -150,6 +163,12 @@ func New(lbID, gatewayURL, accessKey, secretKey, region string) (*Agent, error)
 		reloadNginxFn: reloadNginx,
 		statsFn:       queryHAProxyStats,
 		probeFn:       probeHealthTargets,
+		netcfgPath:    netcfgPath,
+		sysNetDir:     sysClassNet,
+		probeWait:     probeWait,
+		arpProbeFn:    arpProbe,
+		instanceIDFn:  fetchInstanceID,
+		probeStates:   map[string]*nicProbeState{},
 	}, nil
 }
 
@@ -207,7 +226,7 @@ func (a *Agent) tick() {
 		}
 	}
 
-	resp, err := a.sendHeartbeat(servers)
+	resp, err := a.sendHeartbeat(servers, a.dataPathEvidence())
 	if err != nil {
 		// Include the dial target so the serial console disambiguates a
 		// transport failure (never reached AWSGW — "send request: dial ...")
@@ -288,7 +307,7 @@ func (a *Agent) enginePaths(engine string) (configPath, pidPath, certDir string,
 }
 
 // sendHeartbeat sends a heartbeat with health report to the gateway.
-func (a *Agent) sendHeartbeat(servers []ServerStatus) (*heartbeatResponse, error) {
+func (a *Agent) sendHeartbeat(servers []ServerStatus, ev dataPathEvidence) (*heartbeatResponse, error) {
 	params := url.Values{}
 	params.Set("Action", "LBAgentHeartbeat")
 	params.Set("Version", "2015-12-01")
@@ -300,6 +319,7 @@ func (a *Agent) sendHeartbeat(servers []ServerStatus) (*heartbeatResponse, error
 		params.Set("Servers.member."+idx+".Server", s.Server)
 		params.Set("Servers.member."+idx+".Status", s.Status)
 	}
+	ev.addTo(params)
 
 	body, err := a.signedPost(params)
 	if err != nil {

spinifex/agents/elbv2/arp_linux.go

@@ -0,0 +1,62 @@
+//go:build linux
+
+package elbv2
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"syscall"
+	"time"
+)
+
+// arpProbe sends one ARP request for gateway out of iface on an AF_PACKET
+// socket bound to it and waits for the gateway's reply on the same socket.
+func arpProbe(iface string, mac net.HardwareAddr, src, gateway netip.Addr, wait time.Duration) error {
+	ifi, err := net.InterfaceByName(iface)
+	if err != nil {
+		return fmt.Errorf("interface %s: %w", iface, err)
+	}
+
+	proto := htons(syscall.ETH_P_ARP)
+	fd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, int(proto))
+	if err != nil {
+		return fmt.Errorf("packet socket: %w", err)
+	}
+	defer syscall.Close(fd)
+
+	addr := &syscall.SockaddrLinklayer{Protocol: proto, Ifindex: ifi.Index}
+	if err := syscall.Bind(fd, addr); err != nil {
+		return fmt.Errorf("bind %s: %w", iface, err)
+	}
+
+	deadline := time.Now().Add(wait)
+	if err := syscall.Sendto(fd, buildARPRequest(mac, src, gateway), 0, addr); err != nil {
+		return fmt.Errorf("send ARP request: %w", err)
+	}
+
+	buf := make([]byte, 128)
+	for {
+		left := time.Until(deadline)
+		if left <= 0 {
+			return errARPNoReply
+		}
+		tv := syscall.NsecToTimeval(int64(left))
+		if err := syscall.SetsockoptTimeval(fd, syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, &tv); err != nil {
+			return fmt.Errorf("set receive deadline: %w", err)
+		}
+		n, _, err := syscall.Recvfrom(fd, buf, 0)
+		if err != nil {
+			if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EINTR) {
+				continue
+			}
+			return fmt.Errorf("receive ARP reply: %w", err)
+		}
+		if isARPReplyFrom(buf[:n], mac, gateway) {
+			return nil
+		}
+	}
+}
+
+func htons(v uint16) uint16 { return v<<8 | v>>8 }

spinifex/agents/elbv2/arp_other.go

@@ -0,0 +1,15 @@
+//go:build !linux
+
+package elbv2
+
+import (
+	"errors"
+	"net"
+	"net/netip"
+	"time"
+)
+
+// arpProbe is only meaningful inside the Linux guest.
+func arpProbe(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+	return errors.New("ARP probe is only supported on linux")
+}

spinifex/agents/elbv2/datapath.go

@@ -0,0 +1,303 @@
+package elbv2
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	// netcfgPath is the fw_cfg blob the microVM init script reads to configure NICs.
+	netcfgPath = "/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw"
+
+	sysClassNet = "/sys/class/net"
+
+	// probeFailureThreshold is the number of consecutive missed probes after
+	// which a NIC is reported as failing (about 15 s at the poll interval).
+	probeFailureThreshold = 3
+
+	// arpExchangeWait bounds one ARP exchange. probeWait bounds how long a tick
+	// waits for all probes, so a stuck probe never delays the heartbeat.
+	arpExchangeWait = time.Second
+	probeWait       = 1500 * time.Millisecond
+)
+
+// nicConfig is one NIC of the netcfg blob that has a gateway to probe.
+type nicConfig struct {
+	MAC     string
+	Addr    netip.Addr
+	Gateway netip.Addr
+}
+
+// NICReport is the per-NIC data-path evidence carried in the heartbeat.
+type NICReport struct {
+	MAC                 string
+	ProbeOK             bool
+	ConsecutiveFailures int64
+	RxPackets           int64
+	TxPackets           int64
+	RxDropped           int64
+	TxDropped           int64
+}
+
+// arpProbeFn sends one ARP request for gateway out of iface and returns nil
+// when the gateway's reply arrives within timeout.
+type arpProbeFn func(iface string, mac net.HardwareAddr, src, gateway netip.Addr, timeout time.Duration) error
+
+// parseNetcfg reads the KEY=value netcfg blob and returns the NICs that have a
+// gateway, in NIC order. The mgmt NIC carries no gateway and is skipped, as
+// is any NIC without a usable MAC, address or gateway.
+func parseNetcfg(blob []byte) []nicConfig {
+	type raw struct{ mac, cidr, gw string }
+	byIndex := map[int]*raw{}
+
+	sc := bufio.NewScanner(bytes.NewReader(blob))
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		key, val, ok := strings.Cut(line, "=")
+		if !ok || !strings.HasPrefix(key, "NIC") {
+			continue
+		}
+		idxStr, field, ok := strings.Cut(strings.TrimPrefix(key, "NIC"), "_")
+		if !ok {
+			continue
+		}
+		idx, err := strconv.Atoi(idxStr)
+		if err != nil {
+			continue
+		}
+		r := byIndex[idx]
+		if r == nil {
+			r = &raw{}
+			byIndex[idx] = r
+		}
+		switch field {
+		case "MAC":
+			r.mac = strings.ToLower(val)
+		case "CIDR":
+			r.cidr = val
+		case "GW":
+			r.gw = val
+		}
+	}
+
+	idxs := make([]int, 0, len(byIndex))
+	for i := range byIndex {
+		idxs = append(idxs, i)
+	}
+	sort.Ints(idxs)
+
+	var out []nicConfig
+	for _, i := range idxs {
+		r := byIndex[i]
+		if r.mac == "" || r.cidr == "" || r.gw == "" {
+			continue
+		}
+		prefix, err := netip.ParsePrefix(r.cidr)
+		if err != nil || !prefix.Addr().Is4() {
+			continue
+		}
+		gw, err := netip.ParseAddr(r.gw)
+		if err != nil || !gw.Is4() {
+			continue
+		}
+		out = append(out, nicConfig{MAC: r.mac, Addr: prefix.Addr(), Gateway: gw})
+	}
+	return out
+}
+
+// ifaceByMAC returns the interface under sysNetDir whose address is mac.
+func ifaceByMAC(sysNetDir, mac string) (string, error) {
+	entries, err := os.ReadDir(sysNetDir)
+	if err != nil {
+		return "", err
+	}
+	for _, e := range entries {
+		b, err := os.ReadFile(filepath.Join(sysNetDir, e.Name(), "address"))
+		if err != nil {
+			continue
+		}
+		if strings.EqualFold(strings.TrimSpace(string(b)), mac) {
+			return e.Name(), nil
+		}
+	}
+	return "", fmt.Errorf("no interface with MAC %s", mac)
+}
+
+// readCounters fills the sysfs statistics of iface into r.
+func readCounters(sysNetDir, iface string, r *NICReport) {
+	read := func(name string) int64 {
+		b, err := os.ReadFile(filepath.Join(sysNetDir, iface, "statistics", name))
+		if err != nil {
+			return 0
+		}
+		v, _ := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
+		return v
+	}
+	r.RxPackets = read("rx_packets")
+	r.TxPackets = read("tx_packets")
+	r.RxDropped = read("rx_dropped")
+	r.TxDropped = read("tx_dropped")
+}
+
+var errARPNoReply = errors.New("no ARP reply from the gateway")
+
+// buildARPRequest returns the Ethernet frame asking who has target.
+func buildARPRequest(mac net.HardwareAddr, src, target netip.Addr) []byte {
+	f := make([]byte, 0, 42)
+	f = append(f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff)
+	f = append(f, mac...)
+	f = binary.BigEndian.AppendUint16(f, 0x0806)
+	f = binary.BigEndian.AppendUint16(f, 1)      // Ethernet
+	f = binary.BigEndian.AppendUint16(f, 0x0800) // IPv4
+	f = append(f, 6, 4)
+	f = binary.BigEndian.AppendUint16(f, 1) // request
+	f = append(f, mac...)
+	s := src.As4()
+	f = append(f, s[:]...)
+	f = append(f, 0, 0, 0, 0, 0, 0)
+	t := target.As4()
+	f = append(f, t[:]...)
+	return f
+}
+
+// isARPReplyFrom reports whether frame is an ARP reply sent by sender to mac.
+func isARPReplyFrom(frame []byte, mac net.HardwareAddr, sender netip.Addr) bool {
+	if len(frame) < 42 || binary.BigEndian.Uint16(frame[12:14]) != 0x0806 {
+		return false
+	}
+	if binary.BigEndian.Uint16(frame[20:22]) != 2 || !bytes.Equal(frame[32:38], mac) {
+		return false
+	}
+	s := sender.As4()
+	return bytes.Equal(frame[28:32], s[:])
+}
+
+// nicProbeState carries the miss counter of one NIC across ticks. A probe that
+// has not returned is still in flight, and counts as a miss without starting
+// another goroutine behind it.
+type nicProbeState struct {
+	mu       sync.Mutex
+	failures int64
+	inflight bool
+}
+
+type probeResult struct {
+	ok  bool
+	set bool
+}
+
+// probeNICs probes every gateway-bearing data NIC once, in parallel, and
+// returns within probeWait however long a probe blocks.
+func (a *Agent) probeNICs() []NICReport {
+	blob, err := os.ReadFile(a.netcfgPath)
+	if err != nil {
+		return nil
+	}
+	nics := parseNetcfg(blob)
+	if len(nics) == 0 {
+		return nil
+	}
+
+	reports := make([]NICReport, len(nics))
+	results := make([]chan probeResult, len(nics))
+	for i, n := range nics {
+		results[i] = make(chan probeResult, 1)
+		st := a.probeState(n.MAC)
+
+		st.mu.Lock()
+		busy := st.inflight
+		if !busy {
+			st.inflight = true
+		}
+		st.mu.Unlock()
+		if busy {
+			results[i] <- probeResult{set: true}
+			continue
+		}
+
+		go func(n nicConfig, st *nicProbeState, out chan<- probeResult) {
+			ok := a.probeOne(n)
+			st.mu.Lock()
+			st.inflight = false
+			st.mu.Unlock()
+			out <- probeResult{ok: ok, set: true}
+		}(n, st, results[i])
+	}
+
+	deadline := time.NewTimer(a.probeWait)
+	defer deadline.Stop()
+	expired := false
+	for i, n := range nics {
+		var res probeResult
+		if expired {
+			select {
+			case res = <-results[i]:
+			default:
+			}
+		} else {
+			select {
+			case res = <-results[i]:
+			case <-deadline.C:
+				// Out of time: an unfinished probe is a miss for this tick.
+				expired = true
+			}
+		}
+
+		st := a.probeState(n.MAC)
+		st.mu.Lock()
+		if res.ok {
+			st.failures = 0
+		} else {
+			st.failures++
+		}
+		failures := st.failures
+		st.mu.Unlock()
+
+		r := NICReport{
+			MAC:                 n.MAC,
+			ProbeOK:             failures < probeFailureThreshold,
+			ConsecutiveFailures: failures,
+		}
+		if iface, err := ifaceByMAC(a.sysNetDir, n.MAC); err == nil {
+			readCounters(a.sysNetDir, iface, &r)
+		}
+		reports[i] = r
+	}
+	return reports
+}
+
+// probeOne runs one ARP exchange for n, false on any failure.
+func (a *Agent) probeOne(n nicConfig) bool {
+	iface, err := ifaceByMAC(a.sysNetDir, n.MAC)
+	if err != nil {
+		return false
+	}
+	hw, err := net.ParseMAC(n.MAC)
+	if err != nil {
+		return false
+	}
+	return a.arpProbeFn(iface, hw, n.Addr, n.Gateway, arpExchangeWait) == nil
+}
+
+func (a *Agent) probeState(mac string) *nicProbeState {
+	a.probeMu.Lock()
+	defer a.probeMu.Unlock()
+	st := a.probeStates[mac]
+	if st == nil {
+		st = &nicProbeState{}
+		a.probeStates[mac] = st
+	}
+	return st
+}

spinifex/agents/elbv2/datapath_test.go

@@ -0,0 +1,484 @@
+package elbv2
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+const (
+	testDataMAC = "02:00:00:aa:00:01"
+	testMgmtMAC = "02:00:00:aa:00:02"
+	testNetcfg  = "NIC0_MAC=" + testDataMAC + "\nNIC0_CIDR=10.0.1.5/24\nNIC0_GW=10.0.1.1\nNIC0_DEFAULT=1\n" +
+		"NIC1_MAC=" + testMgmtMAC + "\nNIC1_CIDR=10.255.0.9/24\nNIC1_DEFAULT=0\n"
+)
+
+// datapathAgent returns a test agent whose netcfg and sysfs live in a temp dir.
+func datapathAgent(t *testing.T, gwURL, netcfg string) *Agent {
+	t.Helper()
+	a := newTestAgent(t, gwURL)
+	dir := t.TempDir()
+
+	a.netcfgPath = filepath.Join(dir, "netcfg")
+	if err := os.WriteFile(a.netcfgPath, []byte(netcfg), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	a.sysNetDir = filepath.Join(dir, "net")
+	for name, mac := range map[string]string{"ens3": testDataMAC, "ens4": testMgmtMAC} {
+		stats := filepath.Join(a.sysNetDir, name, "statistics")
+		if err := os.MkdirAll(stats, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(a.sysNetDir, name, "address"), []byte(mac+"\n"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		for file, v := range map[string]string{"rx_packets": "100", "tx_packets": "200", "rx_dropped": "3", "tx_dropped": "4"} {
+			if err := os.WriteFile(filepath.Join(stats, file), []byte(v+"\n"), 0o600); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	a.instanceIDFn = func() (string, error) { return "i-0123456789abcdef0", nil }
+	return a
+}
+
+func TestParseNetcfg_SkipsMgmtNICAndMissingGateway(t *testing.T) {
+	blob := testNetcfg +
+		"NIC2_MAC=02:00:00:aa:00:03\nNIC2_CIDR=10.0.2.5/24\nNIC2_GW=10.0.2.1\nNIC2_DEFAULT=0\n" +
+		"NIC3_MAC=02:00:00:aa:00:04\nNIC3_DHCP=1\nNIC3_DEFAULT=0\n" +
+		"NIC4_CIDR=10.0.4.5/24\nNIC4_GW=10.0.4.1\n" +
+		"NIC5_MAC=02:00:00:aa:00:06\nNIC5_CIDR=not-a-cidr\nNIC5_GW=10.0.5.1\n"
+
+	got := parseNetcfg([]byte(blob))
+	if len(got) != 2 {
+		t.Fatalf("got %d NICs, want the 2 with a gateway: %+v", len(got), got)
+	}
+	if got[0].MAC != testDataMAC || got[0].Gateway != netip.MustParseAddr("10.0.1.1") || got[0].Addr != netip.MustParseAddr("10.0.1.5") {
+		t.Errorf("first NIC = %+v", got[0])
+	}
+	if got[1].MAC != "02:00:00:aa:00:03" || got[1].Gateway != netip.MustParseAddr("10.0.2.1") {
+		t.Errorf("second NIC = %+v", got[1])
+	}
+}
+
+func TestParseNetcfg_OrdersByNICIndexAndLowercasesMAC(t *testing.T) {
+	blob := "NIC10_MAC=02:00:00:AA:00:0A\nNIC10_CIDR=10.0.9.5/24\nNIC10_GW=10.0.9.1\n" +
+		"NIC2_MAC=02:00:00:AA:00:02\nNIC2_CIDR=10.0.2.5/24\nNIC2_GW=10.0.2.1\n"
+	got := parseNetcfg([]byte(blob))
+	if len(got) != 2 || got[0].MAC != "02:00:00:aa:00:02" || got[1].MAC != "02:00:00:aa:00:0a" {
+		t.Fatalf("order or case wrong: %+v", got)
+	}
+}
+
+func TestIfaceByMACAndCounters(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+
+	iface, err := ifaceByMAC(a.sysNetDir, "02:00:00:AA:00:01")
+	if err != nil || iface != "ens3" {
+		t.Fatalf("iface = %q, err = %v", iface, err)
+	}
+	if _, err := ifaceByMAC(a.sysNetDir, "02:00:00:ff:ff:ff"); err == nil {
+		t.Error("an unknown MAC must not resolve to an interface")
+	}
+
+	var r NICReport
+	readCounters(a.sysNetDir, iface, &r)
+	if r.RxPackets != 100 || r.TxPackets != 200 || r.RxDropped != 3 || r.TxDropped != 4 {
+		t.Errorf("counters = %+v", r)
+	}
+}
+
+func TestARPFrames_RequestShapeAndReplyMatching(t *testing.T) {
+	mac, _ := net.ParseMAC(testDataMAC)
+	src, gw := netip.MustParseAddr("10.0.1.5"), netip.MustParseAddr("10.0.1.1")
+
+	req := buildARPRequest(mac, src, gw)
+	if len(req) != 42 || req[12] != 0x08 || req[13] != 0x06 || req[21] != 1 {
+		t.Fatalf("request frame malformed: % x", req)
+	}
+	if string(req[38:42]) != string(gw.AsSlice()) {
+		t.Errorf("target IP not the gateway: % x", req[38:42])
+	}
+
+	reply := make([]byte, 42)
+	copy(reply[0:6], mac)
+	reply[12], reply[13] = 0x08, 0x06
+	reply[21] = 2
+	copy(reply[28:32], gw.AsSlice())
+	copy(reply[32:38], mac)
+	if !isARPReplyFrom(reply, mac, gw) {
+		t.Error("a reply from the gateway to our MAC must match")
+	}
+	if isARPReplyFrom(reply, mac, netip.MustParseAddr("10.0.1.99")) {
+		t.Error("a reply from another host must not match")
+	}
+	reply[21] = 1
+	if isARPReplyFrom(reply, mac, gw) {
+		t.Error("a request must not count as a reply")
+	}
+	if isARPReplyFrom(reply[:20], mac, gw) {
+		t.Error("a short frame must not match")
+	}
+}
+
+func TestProbeNICs_ProbesOnlyTheGatewayNIC(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	var mu sync.Mutex
+	var probed []string
+	a.arpProbeFn = func(iface string, _ net.HardwareAddr, src, gw netip.Addr, _ time.Duration) error {
+		mu.Lock()
+		defer mu.Unlock()
+		probed = append(probed, fmt.Sprintf("%s %s>%s", iface, src, gw))
+		return nil
+	}
+
+	reports := a.probeNICs()
+	if len(reports) != 1 || reports[0].MAC != testDataMAC || !reports[0].ProbeOK || reports[0].ConsecutiveFailures != 0 {
+		t.Fatalf("reports = %+v", reports)
+	}
+	if len(probed) != 1 || probed[0] != "ens3 10.0.1.5>10.0.1.1" {
+		t.Errorf("probed = %v, want only the data NIC", probed)
+	}
+	if reports[0].RxPackets != 100 || reports[0].TxDropped != 4 {
+		t.Errorf("counters missing from report: %+v", reports[0])
+	}
+}
+
+func TestProbeNICs_TwoMissesThenSuccessNeverReportsFailing(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	results := []error{errARPNoReply, errARPNoReply, nil, errARPNoReply, errARPNoReply, nil}
+	var n atomic.Int32
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+		return results[n.Add(1)-1]
+	}
+
+	for i, want := range []int64{1, 2, 0, 1, 2, 0} {
+		r := a.probeNICs()
+		if len(r) != 1 || !r[0].ProbeOK || r[0].ConsecutiveFailures != want {
+			t.Fatalf("tick %d: report = %+v, want ok with %d failures", i, r, want)
+		}
+	}
+}
+
+func TestProbeNICs_ThreeMissesReportFailingAndRecoveryClears(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	var fail atomic.Bool
+	fail.Store(true)
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+		if fail.Load() {
+			return errARPNoReply
+		}
+		return nil
+	}
+
+	for i := 1; i <= 4; i++ {
+		r := a.probeNICs()
+		wantOK := i < 3 // failing from the third consecutive miss
+		if r[0].ProbeOK != wantOK || r[0].ConsecutiveFailures != int64(i) {
+			t.Fatalf("miss %d: report = %+v, want ProbeOK=%v", i, r[0], wantOK)
+		}
+	}
+
+	fail.Store(false)
+	if r := a.probeNICs(); !r[0].ProbeOK || r[0].ConsecutiveFailures != 0 {
+		t.Errorf("a successful probe must clear the failure: %+v", r[0])
+	}
+}
+
+func TestProbeNICs_BlockedProbeIsBoundedAndCountsAsMiss(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	a.probeWait = 50 * time.Millisecond
+	release := make(chan struct{})
+	defer close(release)
+	var calls atomic.Int32
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+		calls.Add(1)
+		<-release
+		return nil
+	}
+
+	for i := 1; i <= 3; i++ {
+		start := time.Now()
+		r := a.probeNICs()
+		if d := time.Since(start); d > time.Second {
+			t.Fatalf("tick %d waited %s on a stuck probe", i, d)
+		}
+		if len(r) != 1 || r[0].ConsecutiveFailures != int64(i) {
+			t.Fatalf("tick %d: report = %+v, want %d misses", i, r, i)
+		}
+	}
+	if got := calls.Load(); got != 1 {
+		t.Errorf("a stuck probe was started %d times, want 1 (no goroutine pile-up)", got)
+	}
+}
+
+func TestTick_BlockedProbeDoesNotDelayHeartbeatBeyondOneTick(t *testing.T) {
+	var got atomic.Pointer[url.Values]
+	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseForm()
+		if r.FormValue("Action") == "LBAgentHeartbeat" {
+			v := r.Form
+			got.Store(&v)
+		}
+		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
+	}))
+	defer gw.Close()
+
+	a := datapathAgent(t, gw.URL, testNetcfg)
+	a.localConfigHash = "h1"
+	a.probeWait = 50 * time.Millisecond
+	release := make(chan struct{})
+	defer close(release)
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+		<-release
+		return nil
+	}
+
+	start := time.Now()
+	a.tick()
+	if d := time.Since(start); d >= pollInterval {
+		t.Fatalf("tick took %s with a stuck probe, want under one %s tick", d, pollInterval)
+	}
+	if got.Load() == nil {
+		t.Fatal("the heartbeat was never sent")
+	}
+}
+
+func TestHeartbeat_EncodesEvidenceWhenPresent(t *testing.T) {
+	var got atomic.Pointer[url.Values]
+	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseForm()
+		v := r.Form
+		got.Store(&v)
+		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
+	}))
+	defer gw.Close()
+
+	a := datapathAgent(t, gw.URL, testNetcfg)
+	a.localConfigHash = "h1"
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return errARPNoReply }
+
+	a.tick()
+	v := got.Load()
+	if v == nil {
+		t.Fatal("no heartbeat received")
+	}
+	want := map[string]string{
+		"InstanceID":                        "i-0123456789abcdef0",
+		"NICs.member.1.MAC":                 testDataMAC,
+		"NICs.member.1.ProbeOK":             "true",
+		"NICs.member.1.ConsecutiveFailures": "1",
+		"NICs.member.1.RxPackets":           "100",
+		"NICs.member.1.TxPackets":           "200",
+		"NICs.member.1.RxDropped":           "3",
+		"NICs.member.1.TxDropped":           "4",
+	}
+	for k, w := range want {
+		if g := v.Get(k); g != w {
+			t.Errorf("%s = %q, want %q", k, g, w)
+		}
+	}
+	if v.Get("NICs.member.2.MAC") != "" {
+		t.Error("the mgmt NIC must not be reported")
+	}
+}
+
+func TestHeartbeat_OmitsEvidenceWithoutNetcfg(t *testing.T) {
+	var got atomic.Pointer[url.Values]
+	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseForm()
+		v := r.Form
+		got.Store(&v)
+		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
+	}))
+	defer gw.Close()
+
+	a := datapathAgent(t, gw.URL, testNetcfg)
+	a.localConfigHash = "h1"
+	a.netcfgPath = filepath.Join(t.TempDir(), "absent")
+	a.instanceIDFn = func() (string, error) { return "", errors.New("must not be asked without NICs") }
+
+	a.tick()
+	v := got.Load()
+	if v == nil {
+		t.Fatal("no heartbeat received")
+	}
+	for k := range *v {
+		if k == "InstanceID" || len(k) >= 4 && k[:4] == "NICs" {
+			t.Errorf("unexpected evidence parameter %s", k)
+		}
+	}
+}
+
+func TestDataPathEvidence_InstanceIDRetriedUntilKnown(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return nil }
+	var calls atomic.Int32
+	a.instanceIDFn = func() (string, error) {
+		if calls.Add(1) < 3 {
+			return "", errors.New("metadata not ready")
+		}
+		return "i-late", nil
+	}
+
+	// A failure backs the lookup off for imdsRetryTicks-1 ticks.
+	ticks := func(n int) string {
+		var id string
+		for range n {
+			id = a.dataPathEvidence().InstanceID
+		}
+		return id
+	}
+	if got := ticks(imdsRetryTicks); got != "" || calls.Load() != 1 {
+		t.Fatalf("after %d ticks: id %q, calls %d; want no id and one lookup", imdsRetryTicks, got, calls.Load())
+	}
+	if got := ticks(imdsRetryTicks); got != "" || calls.Load() != 2 {
+		t.Fatalf("second window: id %q, calls %d; want no id and two lookups", got, calls.Load())
+	}
+	if got := ticks(imdsRetryTicks); got != "i-late" || calls.Load() != 3 {
+		t.Fatalf("third window: id %q, calls %d; want i-late and three lookups", got, calls.Load())
+	}
+	ticks(imdsRetryTicks)
+	if calls.Load() != 3 {
+		t.Errorf("metadata asked %d times, want 3 (stop once known)", calls.Load())
+	}
+}
+
+func TestTick_BlockedInstanceIDLookupDoesNotDelayHeartbeat(t *testing.T) {
+	var got atomic.Pointer[url.Values]
+	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseForm()
+		if r.FormValue("Action") == "LBAgentHeartbeat" {
+			v := r.Form
+			got.Store(&v)
+		}
+		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
+	}))
+	defer gw.Close()
+
+	a := datapathAgent(t, gw.URL, testNetcfg)
+	a.localConfigHash = "h1"
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return nil }
+	release := make(chan struct{})
+	defer close(release)
+	var calls atomic.Int32
+	a.instanceIDFn = func() (string, error) {
+		calls.Add(1)
+		<-release
+		return "", errors.New("unreachable")
+	}
+
+	for i := range 3 {
+		start := time.Now()
+		a.tick()
+		if d := time.Since(start); d >= pollInterval {
+			t.Fatalf("tick %d took %s with IMDS blocked, want under one %s tick", i, d, pollInterval)
+		}
+	}
+	v := got.Load()
+	if v == nil {
+		t.Fatal("the heartbeat was never sent")
+	}
+	if v.Get("InstanceID") != "" {
+		t.Error("InstanceID must be omitted while the lookup fails")
+	}
+	if v.Get("NICs.member.1.MAC") == "" {
+		t.Error("NIC evidence must still be sent")
+	}
+	if calls.Load() != 1 {
+		t.Errorf("blocked lookup started %d times in 3 ticks, want 1 (backoff)", calls.Load())
+	}
+}
+
+func TestFetchInstanceID_SharesOneDeadline(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(3 * imdsWait)
+	}))
+	defer srv.Close()
+	old := imdsBase
+	imdsBase = srv.URL + "/latest"
+	defer func() { imdsBase = old }()
+
+	start := time.Now()
+	if _, err := fetchInstanceID(); err == nil {
+		t.Fatal("a hung metadata service must fail the lookup")
+	}
+	if d := time.Since(start); d > imdsWait+imdsWait/2 {
+		t.Fatalf("lookup took %s, want about one %s deadline", d, imdsWait)
+	}
+}
+
+func TestARPProbe_NoResponderFailsClosed(t *testing.T) {
+	mac, _ := net.ParseMAC(testDataMAC)
+	// Nobody answers an ARP request sent on loopback, so a probe that returned
+	// nil here would mask a dead data path. It fails either on the socket (no
+	// CAP_NET_RAW) or on the missing reply.
+	err := arpProbe("lo", mac, netip.MustParseAddr("10.9.9.2"), netip.MustParseAddr("10.9.9.1"), 100*time.Millisecond)
+	if err == nil {
+		t.Fatal("a probe with no responder must fail")
+	}
+	if err := arpProbe("no-such-iface0", mac, netip.MustParseAddr("10.9.9.2"), netip.MustParseAddr("10.9.9.1"), 100*time.Millisecond); err == nil {
+		t.Fatal("a probe on a missing interface must fail")
+	}
+}
+
+func TestFetchInstanceID(t *testing.T) {
+	serve := func(tokenStatus, idStatus int, wantToken string) *httptest.Server {
+		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			switch {
+			case r.Method == http.MethodPut && r.URL.Path == "/latest/api/token":
+				w.WriteHeader(tokenStatus)
+				fmt.Fprint(w, "tok123")
+			case r.URL.Path == "/latest/meta-data/instance-id":
+				if got := r.Header.Get("X-Aws-Ec2-Metadata-Token"); got != wantToken {
+					t.Errorf("token header = %q, want %q", got, wantToken)
+				}
+				w.WriteHeader(idStatus)
+				fmt.Fprint(w, " i-abc123 \n")
+			default:
+				http.NotFound(w, r)
+			}
+		}))
+	}
+	saved := imdsBase
+	defer func() { imdsBase = saved }()
+
+	for _, tc := range []struct {
+		name                  string
+		tokenStatus, idStatus int
+		wantToken             string
+		wantID                string
+		wantErr               bool
+	}{
+		{"token then id", http.StatusOK, http.StatusOK, "tok123", "i-abc123", false},
+		{"token refused falls back to v1", http.StatusNotFound, http.StatusOK, "", "i-abc123", false},
+		{"id endpoint error", http.StatusOK, http.StatusServiceUnavailable, "tok123", "", true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			srv := serve(tc.tokenStatus, tc.idStatus, tc.wantToken)
+			defer srv.Close()
+			imdsBase = srv.URL + "/latest"
+			id, err := fetchInstanceID()
+			if (err != nil) != tc.wantErr || id != tc.wantID {
+				t.Fatalf("id = %q, err = %v", id, err)
+			}
+		})
+	}
+
+	imdsBase = "http://127.0.0.1:1/latest"
+	if _, err := fetchInstanceID(); err == nil {
+		t.Error("an unreachable metadata service must fail")
+	}
+}

spinifex/agents/elbv2/evidence.go

@@ -0,0 +1,133 @@
+package elbv2
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	imdsWait       = time.Second
+	imdsRetryTicks = 6
+)
+
+// imdsBase is the in-VM metadata endpoint (a variable so tests can point it at a fake).
+var imdsBase = "http://169.254.169.254/latest"
+
+// dataPathEvidence is the optional part of a heartbeat: which VM the agent runs
+// in and what each data NIC's gateway probe saw. An empty value adds nothing,
+// which is what an agent without a netcfg blob sends.
+type dataPathEvidence struct {
+	InstanceID string
+	NICs       []NICReport
+}
+
+// addTo encodes the evidence as additional heartbeat query parameters.
+func (e dataPathEvidence) addTo(params url.Values) {
+	if e.InstanceID != "" {
+		params.Set("InstanceID", e.InstanceID)
+	}
+	for i, n := range e.NICs {
+		p := "NICs.member." + strconv.Itoa(i+1) + "."
+		params.Set(p+"MAC", n.MAC)
+		params.Set(p+"ProbeOK", strconv.FormatBool(n.ProbeOK))
+		params.Set(p+"ConsecutiveFailures", strconv.FormatInt(n.ConsecutiveFailures, 10))
+		params.Set(p+"RxPackets", strconv.FormatInt(n.RxPackets, 10))
+		params.Set(p+"TxPackets", strconv.FormatInt(n.TxPackets, 10))
+		params.Set(p+"RxDropped", strconv.FormatInt(n.RxDropped, 10))
+		params.Set(p+"TxDropped", strconv.FormatInt(n.TxDropped, 10))
+	}
+}
+
+// dataPathEvidence gathers the probe results and instance ID for one heartbeat.
+// Neither step can fail the heartbeat: missing evidence is simply omitted.
+func (a *Agent) dataPathEvidence() dataPathEvidence {
+	nics := a.probeNICs()
+	if len(nics) == 0 {
+		return dataPathEvidence{}
+	}
+	if a.instanceID == "" {
+		a.lookupInstanceID()
+	}
+	return dataPathEvidence{InstanceID: a.instanceID, NICs: nics}
+}
+
+// lookupInstanceID asks for the instance ID without ever holding up a tick: the
+// call is bounded by imdsWait and, after a failure, retried only every
+// imdsRetryTicks ticks.
+func (a *Agent) lookupInstanceID() {
+	if a.idSkip > 0 {
+		a.idSkip--
+		return
+	}
+	type result struct {
+		id  string
+		err error
+	}
+	ch := make(chan result, 1)
+	go func() {
+		id, err := a.instanceIDFn()
+		ch <- result{id, err}
+	}()
+	timer := time.NewTimer(imdsWait)
+	defer timer.Stop()
+	select {
+	case r := <-ch:
+		if r.err == nil && r.id != "" {
+			a.instanceID = r.id
+			return
+		}
+	case <-timer.C:
+	}
+	a.idSkip = imdsRetryTicks - 1
+}
+
+// fetchInstanceID reads the instance ID from the in-VM metadata service. The
+// token request and the read share one deadline.
+func fetchInstanceID() (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), imdsWait)
+	defer cancel()
+	c := &http.Client{}
+
+	var token string
+	if req, err := http.NewRequestWithContext(ctx, http.MethodPut, imdsBase+"/api/token", nil); err == nil {
+		req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "60")
+		if resp, err := c.Do(req); err == nil {
+			b, _ := io.ReadAll(io.LimitReader(resp.Body, 256))
+			resp.Body.Close()
+			if resp.StatusCode == http.StatusOK {
+				token = strings.TrimSpace(string(b))
+			}
+		}
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, imdsBase+"/meta-data/instance-id", nil)
+	if err != nil {
+		return "", err
+	}
+	if token != "" {
+		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
+	}
+	resp, err := c.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("instance-id: status %d", resp.StatusCode)
+	}
+	b, err := io.ReadAll(io.LimitReader(resp.Body, 256))
+	if err != nil {
+		return "", err
+	}
+	id := strings.TrimSpace(string(b))
+	if id == "" {
+		return "", fmt.Errorf("instance-id: empty")
+	}
+	return id, nil
+}

spinifex/agents/lambda/hostlink/configdisk.go

@@ -0,0 +1,71 @@
+package hostlink
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"golang.org/x/sys/unix"
+)
+
+// On a VMM without fw_cfg the executor attaches the same entries as a tar
+// archive on a second disk, which the runtime image's init extracts here in
+// the fw_cfg sysfs layout, so the fw_cfg readers work unchanged.
+const (
+	ConfigDiskRoot   = "/spinifex-cfg"
+	ConfigDiskDir    = ConfigDiskRoot + "/by_name/" + lambdav1.FwCfgPrefix
+	ConfigDiskDevice = "/dev/vdb"
+)
+
+// zeroChunk bounds each write while erasing the configuration disk.
+const zeroChunk = 64 << 10
+
+// unmount is a var so tests without privilege can stand in for the tmpfs.
+var unmount = unix.Unmount
+
+// SealConfigDisk overwrites the whole configuration disk with zeros, then
+// removes the extracted tree. The tree stays if the disk cannot be erased, so
+// SealFwCfg's check on root fails closed.
+func SealConfigDisk(device, root string) error {
+	if err := zeroDevice(device); err != nil {
+		return fmt.Errorf("hostlink: erase configuration disk: %w", err)
+	}
+	if err := unmount(root, 0); err != nil && !errors.Is(err, unix.EINVAL) {
+		return fmt.Errorf("hostlink: unmount %s: %w", root, err)
+	}
+	if err := os.RemoveAll(root); err != nil {
+		return fmt.Errorf("hostlink: remove %s: %w", root, err)
+	}
+	return nil
+}
+
+func zeroDevice(device string) error {
+	f, err := os.OpenFile(device, os.O_WRONLY, 0)
+	if err != nil {
+		return err
+	}
+	size, err := f.Seek(0, io.SeekEnd)
+	if err == nil {
+		_, err = f.Seek(0, io.SeekStart)
+	}
+	if err != nil {
+		_ = f.Close()
+		return err
+	}
+	zeros := make([]byte, zeroChunk)
+	for left := size; left > 0; {
+		n := min(left, int64(len(zeros)))
+		if _, err := f.Write(zeros[:n]); err != nil {
+			_ = f.Close()
+			return err
+		}
+		left -= n
+	}
+	if err := f.Sync(); err != nil {
+		_ = f.Close()
+		return err
+	}
+	return f.Close()
+}

spinifex/agents/lambda/hostlink/configdisk_test.go

@@ -0,0 +1,63 @@
+//test:in-package reaches the unexported configDisk and fwCfgTree readers to seal and read a real archive
+
+package hostlink
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// configDisk writes a fake disk holding secret and an extracted tree; the
+// stand-in unmount reports the tree is not a mount, as on a plain directory.
+func configDisk(t *testing.T, secret []byte) (device, root string) {
+	t.Helper()
+	dir := t.TempDir()
+	device = filepath.Join(dir, "vdb")
+	require.NoError(t, os.WriteFile(device, append(bytes.Repeat([]byte{0xa5}, 3*zeroChunk+7), secret...), 0o600))
+	root = fwCfgTree(t)
+	require.NoError(t, os.WriteFile(filepath.Join(root, "by_name", "key"), secret, 0o600))
+	old := unmount
+	unmount = func(string, int) error { return unix.EINVAL }
+	t.Cleanup(func() { unmount = old })
+	return device, root
+}
+
+func TestSealConfigDisk(t *testing.T) {
+	secret := []byte("guest-key-material")
+
+	t.Run("erases the disk and removes the tree", func(t *testing.T) {
+		device, root := configDisk(t, secret)
+		before, err := os.Stat(device)
+		require.NoError(t, err)
+		require.NoError(t, SealFwCfg(root, func() error { return SealConfigDisk(device, root) }))
+
+		disk, err := os.ReadFile(device)
+		require.NoError(t, err)
+		assert.Len(t, disk, int(before.Size()))
+		assert.Equal(t, make([]byte, len(disk)), disk, "configuration disk not fully erased")
+		_, err = os.Stat(root)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+	})
+	t.Run("an unwritable disk keeps the tree and fails closed", func(t *testing.T) {
+		device, root := configDisk(t, secret)
+		require.NoError(t, os.Chmod(device, 0o400))
+		err := SealFwCfg(root, func() error { return SealConfigDisk(device, root) })
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "still readable")
+		_, err = os.Stat(root)
+		assert.NoError(t, err)
+	})
+	t.Run("a failed unmount keeps the tree", func(t *testing.T) {
+		device, root := configDisk(t, secret)
+		unmount = func(string, int) error { return unix.EBUSY }
+		require.Error(t, SealFwCfg(root, func() error { return SealConfigDisk(device, root) }))
+		_, err := os.Stat(root)
+		assert.NoError(t, err)
+	})
+}

spinifex/agents/lambda/hostlink/link.go

@@ -28,11 +28,12 @@ type Request struct {
 	Credential *Credential `json:"credential,omitempty"`
 }
 
-// Credential is one execution-role session. It is never logged or stored.
+// Credential holds one execution-role session and its function environment.
 type Credential struct {
-	AccessKeyID     string `json:"access_key_id"`
-	SecretAccessKey string `json:"secret_access_key"`
-	SessionToken    string `json:"session_token"`
+	AccessKeyID     string            `json:"access_key_id"`
+	SecretAccessKey string            `json:"secret_access_key"`
+	SessionToken    string            `json:"session_token"`
+	Environment     map[string]string `json:"environment,omitempty"`
 }
 
 // LogValue keeps every value out of logs.

spinifex/agents/lambda/hostlink/protocol_testlink.go

@@ -4,5 +4,5 @@ package hostlink
 
 // Protocol is the only ALPN protocol a test-link host and guest offer. It
 // differs from the release protocol because the first frame carries the
-// execution credential, which an older guest would ignore.
-const Protocol = "spinifex-lambda-testlink/2"
+// execution credential and function environment, which an older guest would ignore.
+const Protocol = "spinifex-lambda-testlink/3"

spinifex/domains/acm/store.go

@@ -14,7 +14,6 @@ import (
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -184,7 +183,7 @@ func NewStore(ctx context.Context, nc *nats.Conn, masterKey []byte) (*Store, err
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketACM, KVBucketACMVersion)
+	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketACM, KVBucketACMVersion)
 	if err != nil {
 		return nil, err
 	}

spinifex/domains/admission/quota/bedrock.go

@@ -10,7 +10,7 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"golang.org/x/time/rate"
 )
@@ -171,7 +171,7 @@ func (s *Service) RunBedrockRPMSync(ctx context.Context, js jetstream.JetStream)
 	if s == nil || !s.limits.RequestsPerMinuteEnabled {
 		return
 	}
-	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketBedrockRPMUsage, 1)
+	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketBedrockRPMUsage, 1)
 	if err != nil {
 		slog.Warn("bedrock rpm sync: open usage bucket failed, sync disabled", "err", err)
 		return

spinifex/domains/admission/quota/override.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -187,7 +187,7 @@ func (s *Service) PutAccountQuota(ctx context.Context, accountID string, over Ov
 	over.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
 	over.UpdatedBy = updatedBy
 
-	_, err := kvutil.Update(ctx, s.overrides, accountID, kvutil.CASConfig{
+	_, err := kvstore.Update(ctx, s.overrides, accountID, kvstore.CASConfig{
 		CreateIfAbsent: true,
 		Attempts:       overrideMaxAttempts,
 		Exhausted:      func(string, int) error { return errOverridesContended },
@@ -230,7 +230,7 @@ func (s *Service) clearAccountQuota(ctx context.Context, accountID string) error
 			return nil
 		case errors.Is(err, jetstream.ErrKeyNotFound):
 			return nil
-		case kvutil.IsRevisionConflict(err):
+		case kvstore.IsRevisionConflict(err):
 			// A concurrent write landed between the read and the guarded
 			// delete; retry against whatever revision now holds.
 			continue

spinifex/domains/admission/quota/seal.go

@@ -8,7 +8,7 @@ import (
 
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -73,7 +73,7 @@ func (s *Service) sealBucket(ctx context.Context) (jetstream.KeyValue, error) {
 		slog.ErrorContext(ctx, "Quota override seal has no JetStream client wired to open the override bucket")
 		return nil, errors.New(awserrors.ErrorInternalError)
 	}
-	kv, err := kvutil.GetOrCreateBucket(ctx, s.js, KVBucketAccountQuota, 1)
+	kv, err := kvstore.GetOrCreateBucket(ctx, s.js, KVBucketAccountQuota, 1)
 	if err != nil {
 		slog.ErrorContext(ctx, "Failed to open quota override bucket for seal", "err", err)
 		return nil, errors.New(awserrors.ErrorInternalError)
@@ -136,7 +136,7 @@ func (s *Service) SealAccountOverrides(ctx context.Context, accountID string) (*
 		switch {
 		case writeErr == nil:
 			return &OverrideSeal{Revision: newRev}, nil
-		case kvutil.IsRevisionConflict(writeErr):
+		case kvstore.IsRevisionConflict(writeErr):
 			continue
 		default:
 			slog.ErrorContext(ctx, "Failed to write sealed quota override record", "accountID", accountID, "err", writeErr)

spinifex/domains/ec2/account/seal.go

@@ -8,7 +8,7 @@ import (
 
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -109,7 +109,7 @@ func (s *AccountSettingsServiceImpl) SealAccount(ctx context.Context, accountID
 		switch {
 		case writeErr == nil:
 			return &AccountSettingsSeal{Revision: newRev}, nil
-		case kvutil.IsRevisionConflict(writeErr):
+		case kvstore.IsRevisionConflict(writeErr):
 			continue
 		default:
 			slog.ErrorContext(ctx, "Failed to write sealed account settings record", "accountID", accountID, "err", writeErr)

spinifex/domains/ec2/account/service_impl.go

@@ -13,7 +13,7 @@ import (
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -57,7 +57,7 @@ func NewAccountSettingsServiceImplWithNATS(ctx context.Context, cfg *config.Conf
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	settingsKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketAccountSettings, 10)
+	settingsKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketAccountSettings, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create account settings KV bucket: %w", err)
 	}
@@ -115,13 +115,13 @@ func mapSettingsError(ctx context.Context, accountID string, err error) error {
 		return errors.New(awserrors.ErrorBlocked)
 	case errors.Is(err, errSettingsContended):
 		slog.ErrorContext(ctx, "Account settings CAS retries exhausted under contention", "accountID", accountID)
-	case errors.Is(err, kvutil.ErrRead):
+	case errors.Is(err, kvstore.ErrRead):
 		slog.ErrorContext(ctx, "Failed to read account settings from KV", "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrDecode):
+	case errors.Is(err, kvstore.ErrDecode):
 		slog.ErrorContext(ctx, "Corrupt account settings record in KV", "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrEncode):
+	case errors.Is(err, kvstore.ErrEncode):
 		slog.ErrorContext(ctx, "Failed to marshal account settings record", "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrWrite):
+	case errors.Is(err, kvstore.ErrWrite):
 		slog.ErrorContext(ctx, "Failed to write account settings to KV", "accountID", accountID, "err", err)
 	default:
 		slog.ErrorContext(ctx, "Account settings operation failed", "accountID", accountID, "err", err)
@@ -133,7 +133,7 @@ func mapSettingsError(ctx context.Context, accountID string, err error) error {
 // changed is always true, so every success is a leader-accepted write over an
 // active value; a seal landing between the read and the write fails the CAS.
 func (s *AccountSettingsServiceImpl) mutateSettings(ctx context.Context, accountID string, set func(*AccountSettingsRecord)) error {
-	_, err := kvutil.Update(ctx, s.settingsKV, settingsKey(accountID), kvutil.CASConfig{
+	_, err := kvstore.Update(ctx, s.settingsKV, settingsKey(accountID), kvstore.CASConfig{
 		CreateIfAbsent: true,
 		Exhausted:      func(string, int) error { return errSettingsContended },
 	}, func(rec *AccountSettingsRecord) (bool, error) {

spinifex/domains/ec2/eigw/service_impl.go

@@ -19,7 +19,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -56,7 +56,7 @@ func NewEgressOnlyIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	eigwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketEgressOnlyIGW, 10)
+	eigwKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketEgressOnlyIGW, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketEgressOnlyIGW, err)
 	}
@@ -65,7 +65,7 @@ func NewEgressOnlyIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config
 	}
 
 	// Get or create VPC KV bucket for cross-resource ownership validation
-	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
@@ -90,7 +90,7 @@ func (s *EgressOnlyIGWServiceImpl) CreateEgressOnlyInternetGateway(ctx context.C
 		slog.ErrorContext(ctx, "VPC KV unavailable, cannot verify VPC ownership")
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, *input.VpcId)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, *input.VpcId)); err != nil {
 		slog.WarnContext(ctx, "CreateEgressOnlyInternetGateway: VPC not found for account", "vpcId", *input.VpcId, "accountID", accountID)
 		return nil, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", *input.VpcId)
 	}
@@ -109,7 +109,7 @@ func (s *EgressOnlyIGWServiceImpl) CreateEgressOnlyInternetGateway(ctx context.C
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal Egress-only IGW record: %w", err)
 	}
-	if _, err := s.eigwKV.Put(ctx, kvutil.AccountKey(accountID, eigwID), data); err != nil {
+	if _, err := s.eigwKV.Put(ctx, kvstore.AccountKey(accountID, eigwID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -127,7 +127,7 @@ func (s *EgressOnlyIGWServiceImpl) DeleteEgressOnlyInternetGateway(ctx context.C
 	}
 
 	eigwID := *input.EgressOnlyInternetGatewayId
-	key := kvutil.AccountKey(accountID, eigwID)
+	key := kvstore.AccountKey(accountID, eigwID)
 
 	// Verify the EIGW exists before deleting
 	if _, err := s.eigwKV.Get(ctx, key); err != nil {
@@ -178,7 +178,7 @@ func (s *EgressOnlyIGWServiceImpl) DescribeEgressOnlyInternetGateways(ctx contex
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {

spinifex/domains/ec2/eigw/service_impl_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -23,7 +23,7 @@ func setupTestEIGWService(t *testing.T) *EgressOnlyIGWServiceImpl {
 	// Create VPC KV bucket and register test VPCs so fail-closed ownership checks pass
 	vpcEntries := map[string][]byte{}
 	for _, vpcID := range []string{"vpc-test123", "vpc-tagged", "vpc-tagged2"} {
-		vpcEntries[kvutil.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
+		vpcEntries[kvstore.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
 	}
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, vpcEntries)
 
@@ -312,7 +312,7 @@ func TestCreateEgressOnlyInternetGateway_CrossAccountVPCRejected(t *testing.T) {
 	// Create VPC KV bucket with a VPC owned by testAccountID
 	vpcID := "vpc-alpha456"
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, map[string][]byte{
-		kvutil.AccountKey(testAccountID, vpcID): []byte(`{"vpc_id":"vpc-alpha456","state":"available"}`),
+		kvstore.AccountKey(testAccountID, vpcID): []byte(`{"vpc_id":"vpc-alpha456","state":"available"}`),
 	})
 
 	var err error

spinifex/domains/ec2/eip/allocate_idempotency.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -36,7 +36,7 @@ func (s *EIPServiceImpl) allocateOnce(ctx context.Context, accountID string, all
 	if key == "" || s.idemKV == nil {
 		return alloc()
 	}
-	kvKey := kvutil.AccountKey(accountID, idempotencyKeyHash(key))
+	kvKey := kvstore.AccountKey(accountID, idempotencyKeyHash(key))
 
 	if out := s.cachedAllocation(ctx, kvKey); out != nil {
 		slog.WarnContext(ctx, "AllocateAddress: returning first result for a retried request",

spinifex/domains/ec2/eip/lease_rebind.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/domains/network/projection"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/topology"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -80,7 +80,7 @@ func (s *EIPServiceImpl) AllocationExists(ctx context.Context, allocationID stri
 	}
 	suffix := "." + allocationID
 	for _, k := range keys {
-		if k != kvutil.VersionKey && strings.HasSuffix(k, suffix) {
+		if k != kvstore.VersionKey && strings.HasSuffix(k, suffix) {
 			return true, nil
 		}
 	}
@@ -98,7 +98,7 @@ func (s *EIPServiceImpl) findByAllocationIDAnyAccount(ctx context.Context, alloc
 
 	suffix := "." + allocationID
 	for _, k := range keys {
-		if k == kvutil.VersionKey || !strings.HasSuffix(k, suffix) {
+		if k == kvstore.VersionKey || !strings.HasSuffix(k, suffix) {
 			continue
 		}
 		entry, err := s.eipKV.Get(ctx, k)

spinifex/domains/ec2/eip/service_impl.go

@@ -20,7 +20,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -66,7 +66,7 @@ func NewEIPServiceImpl(ctx context.Context, natsConn *nats.Conn, externalIPAM *e
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	eipKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketEIPs, 10)
+	eipKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketEIPs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketEIPs, err)
 	}
@@ -76,7 +76,7 @@ func NewEIPServiceImpl(ctx context.Context, natsConn *nats.Conn, externalIPAM *e
 
 	// TTL'd, so a failure to create it is not worth refusing to serve EIPs over:
 	// a nil bucket only costs post-completion dedupe.
-	idemKV, err := kvutil.GetOrCreateBucketWithTTL(ctx, js, KVBucketEIPIdempotency, 1, eipIdempotencyTTL)
+	idemKV, err := kvstore.GetOrCreateBucketWithTTL(ctx, js, KVBucketEIPIdempotency, 1, eipIdempotencyTTL)
 	if err != nil {
 		slog.Warn("EIP service: idempotency bucket unavailable; a retried AllocateAddress may allocate twice",
 			"bucket", KVBucketEIPIdempotency, "err", err)
@@ -146,7 +146,7 @@ func (s *EIPServiceImpl) allocateAddress(ctx context.Context, input *ec2.Allocat
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal EIP record: %w", err)
 	}
-	if _, err := s.eipKV.Put(ctx, kvutil.AccountKey(accountID, allocID), data); err != nil {
+	if _, err := s.eipKV.Put(ctx, kvstore.AccountKey(accountID, allocID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -167,7 +167,7 @@ func (s *EIPServiceImpl) ReleaseAddress(ctx context.Context, input *ec2.ReleaseA
 	}
 
 	allocID := *input.AllocationId
-	key := kvutil.AccountKey(accountID, allocID)
+	key := kvstore.AccountKey(accountID, allocID)
 
 	entry, err := s.eipKV.Get(ctx, key)
 	if err != nil {
@@ -206,7 +206,7 @@ func (s *EIPServiceImpl) AssociateAddress(ctx context.Context, input *ec2.Associ
 	}
 
 	allocID := *input.AllocationId
-	key := kvutil.AccountKey(accountID, allocID)
+	key := kvstore.AccountKey(accountID, allocID)
 
 	entry, err := s.eipKV.Get(ctx, key)
 	if err != nil {
@@ -536,7 +536,7 @@ func (s *EIPServiceImpl) findByENI(ctx context.Context, accountID, eniID string)
 		if err := ctx.Err(); err != nil {
 			return nil, err
 		}
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.eipKV.Get(ctx, k)
@@ -590,7 +590,7 @@ func (s *EIPServiceImpl) DescribeAddresses(ctx context.Context, input *ec2.Descr
 
 	var addresses []*ec2.Address
 	for _, k := range keys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -699,7 +699,7 @@ func (s *EIPServiceImpl) DescribeAddressesAttribute(ctx context.Context, input *
 			if id == nil {
 				continue
 			}
-			key := kvutil.AccountKey(accountID, *id)
+			key := kvstore.AccountKey(accountID, *id)
 			entry, err := s.eipKV.Get(ctx, key)
 			if err != nil {
 				continue // not found, skip
@@ -722,7 +722,7 @@ func (s *EIPServiceImpl) DescribeAddressesAttribute(ctx context.Context, input *
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
 		for _, k := range keys {
-			if k == kvutil.VersionKey {
+			if k == kvstore.VersionKey {
 				continue
 			}
 			if !strings.HasPrefix(k, prefix) {
@@ -819,7 +819,7 @@ func (s *EIPServiceImpl) findByAssociationID(ctx context.Context, accountID, ass
 	}
 
 	for _, k := range keys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -857,7 +857,7 @@ func (s *EIPServiceImpl) AssociatedPublicIPForInstance(ctx context.Context, acco
 		return "", false
 	}
 	for _, k := range keys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.eipKV.Get(ctx, k)

spinifex/domains/ec2/igw/service_impl.go

@@ -22,7 +22,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -93,7 +93,7 @@ func NewIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	igwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketIGW, 10)
+	igwKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketIGW, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketIGW, err)
 	}
@@ -102,13 +102,13 @@ func NewIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 	}
 
 	// Get or create VPC KV bucket for cross-resource ownership validation
-	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
 
 	// Read EIP associations so detach can refuse while addresses are mapped.
-	eipKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
+	eipKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get EIP KV bucket: %w", err)
 	}
@@ -142,7 +142,7 @@ func (s *IGWServiceImpl) createIGW(ctx context.Context, input *ec2.CreateInterne
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal IGW record: %w", err)
 	}
-	if _, err := s.igwKV.Put(ctx, kvutil.AccountKey(accountID, igwID), data); err != nil {
+	if _, err := s.igwKV.Put(ctx, kvstore.AccountKey(accountID, igwID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -160,7 +160,7 @@ func (s *IGWServiceImpl) DeleteInternetGateway(ctx context.Context, input *ec2.D
 	}
 
 	igwID := *input.InternetGatewayId
-	key := kvutil.AccountKey(accountID, igwID)
+	key := kvstore.AccountKey(accountID, igwID)
 
 	entry, err := s.igwKV.Get(ctx, key)
 	if err != nil {
@@ -236,7 +236,7 @@ func (s *IGWServiceImpl) DescribeInternetGateways(ctx context.Context, input *ec
 	foundIDs := make(map[string]bool)
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -326,7 +326,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A
 
 	igwID := *input.InternetGatewayId
 	vpcID := *input.VpcId
-	key := kvutil.AccountKey(accountID, igwID)
+	key := kvstore.AccountKey(accountID, igwID)
 
 	entry, err := s.igwKV.Get(ctx, key)
 	if err != nil {
@@ -347,7 +347,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A
 		slog.ErrorContext(ctx, "VPC KV unavailable, cannot verify VPC ownership")
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID)); err != nil {
 		slog.WarnContext(ctx, "AttachInternetGateway: VPC not found for account", "vpcId", vpcID, "accountID", accountID)
 		return nil, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
 	}
@@ -388,7 +388,7 @@ func (s *IGWServiceImpl) DetachInternetGateway(ctx context.Context, input *ec2.D
 
 	igwID := *input.InternetGatewayId
 	vpcID := *input.VpcId
-	key := kvutil.AccountKey(accountID, igwID)
+	key := kvstore.AccountKey(accountID, igwID)
 
 	entry, err := s.igwKV.Get(ctx, key)
 	if err != nil {
@@ -449,7 +449,7 @@ func (s *IGWServiceImpl) DetachInternetGateway(ctx context.Context, input *ec2.D
 // associated into vpcID, whether to an interface or a NAT gateway. The listing
 // is filtered server-side to the account's keys.
 func (s *IGWServiceImpl) vpcHasMappedAddress(ctx context.Context, accountID, vpcID string) (bool, error) {
-	lister, err := s.eipKV.ListKeysFiltered(ctx, kvutil.AccountKey(accountID, "*"))
+	lister, err := s.eipKV.ListKeysFiltered(ctx, kvstore.AccountKey(accountID, "*"))
 	if err != nil {
 		return false, err
 	}
@@ -493,11 +493,11 @@ func (s *IGWServiceImpl) CreateAttachedInternetGateway(ctx context.Context, acco
 	if s.vpcKV == nil {
 		return false, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID)); err != nil {
 		return false, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
 	}
 
-	key := kvutil.AccountKey(accountID, igwID)
+	key := kvstore.AccountKey(accountID, igwID)
 	data, err := json.Marshal(IGWRecord{
 		InternetGatewayId: igwID,
 		VpcId:             vpcID,
@@ -558,7 +558,7 @@ func (s *IGWServiceImpl) AttachmentIntent(ctx context.Context, accountID, vpcID
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.igwKV.Get(ctx, key)
@@ -599,7 +599,7 @@ func (s *IGWServiceImpl) AttachmentIntent(ctx context.Context, accountID, vpcID
 // Writing only on the pending transition keeps a pass from waking the drift loop.
 func MarkAttached(ctx context.Context, kv jetstream.KeyValue, recordKey, vpcID string) error {
 	confirmed := false
-	record, err := kvutil.Update(ctx, kv, recordKey, kvutil.CASConfig{}, func(r *IGWRecord) (bool, error) {
+	record, err := kvstore.Update(ctx, kv, recordKey, kvstore.CASConfig{}, func(r *IGWRecord) (bool, error) {
 		// Reset per attempt: a retried CAS may see a record another writer has
 		// already moved out of pending.
 		confirmed = r.VpcId == vpcID && r.AttachState == AttachStatePending

spinifex/domains/ec2/igw/service_impl_test.go

@@ -13,7 +13,7 @@ import (
 	ec2eip "github.com/mulgadc/spinifex/spinifex/domains/ec2/eip"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -55,7 +55,7 @@ func setupTestIGWService(t *testing.T) (*IGWServiceImpl, *nats.Conn) {
 	// Create VPC KV bucket and register test VPCs so fail-closed ownership checks pass
 	vpcEntries := map[string][]byte{}
 	for _, vpcID := range []string{"vpc-test123", "vpc-other", "vpc-lifecycle", "vpc-event-test"} {
-		vpcEntries[kvutil.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
+		vpcEntries[kvstore.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
 	}
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, vpcEntries)
 
@@ -68,7 +68,7 @@ func setupTestIGWService(t *testing.T) (*IGWServiceImpl, *nats.Conn) {
 // gateway come up. Without it an attach stays pending and reports no attachment.
 func confirmAttach(t *testing.T, svc *IGWServiceImpl, igwID, vpcID string) {
 	t.Helper()
-	require.NoError(t, MarkAttached(context.Background(), svc.igwKV, kvutil.AccountKey(testAccountID, igwID), vpcID))
+	require.NoError(t, MarkAttached(context.Background(), svc.igwKV, kvstore.AccountKey(testAccountID, igwID), vpcID))
 }
 
 func createTestIGW(t *testing.T, svc *IGWServiceImpl) string {
@@ -331,7 +331,7 @@ func TestDescribeInternetGateways_LegacyRecordReportsAttachment(t *testing.T) {
 	svc, _ := setupTestIGWService(t)
 	igwID := createTestIGW(t, svc)
 
-	key := kvutil.AccountKey(testAccountID, igwID)
+	key := kvstore.AccountKey(testAccountID, igwID)
 	entry, err := svc.igwKV.Get(context.Background(), key)
 	require.NoError(t, err)
 	var record IGWRecord
@@ -398,7 +398,7 @@ func TestMarkAttached_NoRewriteWhenAlreadyAttached(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 
-	key := kvutil.AccountKey(testAccountID, igwID)
+	key := kvstore.AccountKey(testAccountID, igwID)
 	confirmAttach(t, svc, igwID, "vpc-test123")
 	first, err := svc.igwKV.Get(context.Background(), key)
 	require.NoError(t, err)
@@ -605,7 +605,7 @@ func TestDetachInternetGateway_MappedPublicAddress(t *testing.T) {
 	putEIP := func(rec ec2eip.EIPRecord) {
 		data, err := json.Marshal(rec)
 		require.NoError(t, err)
-		_, err = svc.eipKV.Put(context.Background(), kvutil.AccountKey(testAccountID, rec.AllocationId), data)
+		_, err = svc.eipKV.Put(context.Background(), kvstore.AccountKey(testAccountID, rec.AllocationId), data)
 		require.NoError(t, err)
 	}
 	putEIP(ec2eip.EIPRecord{AllocationId: "eipalloc-other", AssociationId: "eipassoc-other", VpcId: "vpc-other", State: "associated"})
@@ -614,7 +614,7 @@ func TestDetachInternetGateway_MappedPublicAddress(t *testing.T) {
 	// Another account's address naming the same VPC ID is not this account's dependency.
 	foreign, err := json.Marshal(ec2eip.EIPRecord{AllocationId: "eipalloc-foreign", AssociationId: "eipassoc-foreign", VpcId: "vpc-test123", State: "associated"})
 	require.NoError(t, err)
-	_, err = svc.eipKV.Put(context.Background(), kvutil.AccountKey("999999999999", "eipalloc-foreign"), foreign)
+	_, err = svc.eipKV.Put(context.Background(), kvstore.AccountKey("999999999999", "eipalloc-foreign"), foreign)
 	require.NoError(t, err)
 
 	detach := func() error {
@@ -828,7 +828,7 @@ func TestAttachInternetGateway_CrossAccountVPCRejected(t *testing.T) {
 	require.NoError(t, err)
 
 	vpcID := "vpc-alpha123"
-	_, err = vpcKV.Put(t.Context(), kvutil.AccountKey(testAccountID, vpcID), []byte(`{"vpc_id":"vpc-alpha123","state":"available"}`))
+	_, err = vpcKV.Put(t.Context(), kvstore.AccountKey(testAccountID, vpcID), []byte(`{"vpc_id":"vpc-alpha123","state":"available"}`))
 	require.NoError(t, err)
 
 	// Refresh service to pick up the VPC KV bucket

spinifex/domains/ec2/instance/uefistore/cleanup.go

@@ -10,7 +10,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -72,7 +71,7 @@ func (s *Store) InstanceIDs(ctx context.Context) ([]string, error) {
 
 	var ids []string
 	for key := range lister.Keys() {
-		if key != kvutil.VersionKey {
+		if key != kvstore.VersionKey {
 			ids = append(ids, key)
 		}
 	}

spinifex/domains/ec2/instance/uefistore/uefistore_test.go

@@ -11,7 +11,6 @@ import (
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -60,7 +59,7 @@ func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
 	assert.Equal(t, int64(1), status.History())
 	assert.Zero(t, status.TTL(), "a TTL would expire deletion obligations before they are settled")
 
-	v, err := kvutil.ReadVersion(t.Context(), kv)
+	v, err := kvstore.ReadVersion(t.Context(), kv)
 	require.NoError(t, err)
 	assert.Equal(t, uefistore.BucketVersion, v)
 }

spinifex/domains/ec2/inventory/inventory.go

@@ -0,0 +1,67 @@
+// Package inventory is the EC2-owned read capability for an account's complete
+// instance list, for callers outside the EC2 domain.
+package inventory
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Instances lists every non-terminated instance an account owns, or returns an
+// error. It never returns a shorter list than the truth.
+type Instances interface {
+	ListAccountInstances(ctx context.Context, accountID string) (*ec2.DescribeInstancesOutput, error)
+}
+
+// NATSInstances reads the complete list across nodeIDs over NATS.
+type NATSInstances struct {
+	nc      *nats.Conn
+	nodeIDs []string
+}
+
+var _ Instances = (*NATSInstances)(nil)
+
+// NewNATSInstances returns a reader over the configured node set.
+func NewNATSInstances(nc *nats.Conn, nodeIDs []string) *NATSInstances {
+	return &NATSInstances{nc: nc, nodeIDs: nodeIDs}
+}
+
+// ListAccountInstances passes through the strict listing unchanged: any node
+// that did not answer, or any record that could not be read, is an error.
+func (n *NATSInstances) ListAccountInstances(ctx context.Context, accountID string) (*ec2.DescribeInstancesOutput, error) {
+	return ec2instanceapi.DescribeAccountInstancesStrict(ctx, n.nc, n.nodeIDs, accountID)
+}
+
+// NewENITagReader returns a reader of one ENI's tags straight from its stored
+// record, so an ENI that is absent and one that cannot be read are told apart:
+// a missing key is found=false, a failed read or an undecodable record is an
+// error. accountID is the account the ENI is stored under.
+func NewENITagReader(js jetstream.JetStream, accountID string) func(ctx context.Context, eniID string) (map[string]string, bool, error) {
+	return func(ctx context.Context, eniID string) (map[string]string, bool, error) {
+		kv, err := js.KeyValue(ctx, ec2vpc.KVBucketENIs)
+		if err != nil {
+			return nil, false, fmt.Errorf("open ENI bucket: %w", err)
+		}
+		entry, err := kv.Get(ctx, kvstore.AccountKey(accountID, eniID))
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			return nil, false, nil
+		}
+		if err != nil {
+			return nil, false, fmt.Errorf("read ENI record %s: %w", eniID, err)
+		}
+		var record ec2vpc.ENIRecord
+		if err := json.Unmarshal(entry.Value(), &record); err != nil {
+			return nil, false, fmt.Errorf("decode ENI record %s: %w", eniID, err)
+		}
+		return record.Tags, true, nil
+	}
+}

spinifex/domains/ec2/inventory/inventory_test.go

@@ -0,0 +1,49 @@
+package inventory
+
+import (
+	"encoding/json"
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestListAccountInstancesRefusesAnUnknownNodeSet(t *testing.T) {
+	out, err := NewNATSInstances(nil, nil).ListAccountInstances(t.Context(), "000000000000")
+	require.Error(t, err)
+	assert.Nil(t, out, "an unknown node set must not yield a list")
+}
+
+func TestENITagReaderSeparatesGoneFromUnreadable(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	js := testutil.NewJetStream(t, nc)
+	read := NewENITagReader(js, "000000000000")
+
+	_, _, err := read(t.Context(), "eni-1")
+	require.Error(t, err, "a missing ENI bucket is a read failure, not an absent ENI")
+
+	enis, err := kvstore.GetOrCreateBucket(t.Context(), js, ec2vpc.KVBucketENIs, 10)
+	require.NoError(t, err)
+	good, err := json.Marshal(ec2vpc.ENIRecord{NetworkInterfaceId: "eni-good", Tags: map[string]string{"k": "v"}})
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvstore.AccountKey("000000000000", "eni-good"), good)
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvstore.AccountKey("000000000000", "eni-corrupt"), []byte("{not json"))
+	require.NoError(t, err)
+
+	tags, found, err := read(t.Context(), "eni-good")
+	require.NoError(t, err)
+	assert.True(t, found)
+	assert.Equal(t, map[string]string{"k": "v"}, tags)
+
+	_, found, err = read(t.Context(), "eni-absent")
+	require.NoError(t, err)
+	assert.False(t, found, "a missing key is the ENI gone")
+
+	_, found, err = read(t.Context(), "eni-corrupt")
+	require.Error(t, err, "an undecodable record is a read failure")
+	assert.False(t, found)
+}

spinifex/domains/ec2/launchtemplate/account_cleanup.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -33,7 +33,7 @@ type AccountCleanup struct {
 // Unreadable headers, their bodies and unrecognised entries are left for the
 // owner scan to report. A listing failure is returned and nothing is removed.
 func (s *LaunchTemplateServiceImpl) CleanupAccountResidue(ctx context.Context, accountID string, settledBefore time.Time) (*AccountCleanup, error) {
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list launch-template keys: %w", err)
 	}
@@ -145,7 +145,7 @@ type AccountTemplateDeletes struct {
 // it cannot read or decode is left for the owner scan to report. A listing
 // failure is returned and nothing is deleted.
 func (s *LaunchTemplateServiceImpl) DeleteLiveTemplates(ctx context.Context, accountID string, referenced func(id, name string) string) (*AccountTemplateDeletes, error) {
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list launch-template keys: %w", err)
 	}

spinifex/domains/ec2/launchtemplate/account_scan.go

@@ -9,7 +9,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -37,7 +37,7 @@ type AccountScan struct {
 // Headers and name-index entries are read; version bodies are classified from
 // their keys alone, so template data and user data are never loaded.
 func (s *LaunchTemplateServiceImpl) ScanAccount(ctx context.Context, accountID string) (*AccountScan, error) {
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list launch-template keys: %w", err)
 	}

spinifex/domains/ec2/launchtemplate/account_scan_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit/readonlystore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -45,7 +45,7 @@ func TestScanAccountClassifiesLaunchTemplateRecords(t *testing.T) {
 	// A header removed alone, as a delete interrupted after it leaves the rest.
 	deleted := createWithUserData(t, svc, "deleted")
 	inner := svc.kv
-	require.NoError(t, inner.Delete(t.Context(), kvutil.AccountKey(testAccountID, deleted)))
+	require.NoError(t, inner.Delete(t.Context(), kvstore.AccountKey(testAccountID, deleted)))
 
 	// A create whose header write and clean-up both fail.
 	svc.kv = &faultyKV{KeyValue: inner,
@@ -59,16 +59,16 @@ func TestScanAccountClassifiesLaunchTemplateRecords(t *testing.T) {
 	require.Error(t, err)
 	svc.kv = inner
 
-	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0badbadbadbadbad0"), "{not json")
-	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0badbadbadbadbad0.v1"), `{"version_number":1}`)
-	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0mismatch00000000"), `{"launch_template_id":"lt-0other"}`)
-	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0unreadable000000"), `{"launch_template_id":"lt-0unreadable000000"}`)
+	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0badbadbadbadbad0"), "{not json")
+	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0badbadbadbadbad0.v1"), `{"version_number":1}`)
+	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0mismatch00000000"), `{"launch_template_id":"lt-0other"}`)
+	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0unreadable000000"), `{"launch_template_id":"lt-0unreadable000000"}`)
 	putRaw(t, inner, nameKey(testAccountID, "unreadable"), "lt-0unreadable000000")
-	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0bad.vx"), "{}")
+	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0bad.vx"), "{}")
 
 	recorder := &readRecordingKV{KeyValue: readonlystore.NewKV(t, inner)}
 	svc.kv = &faultyKV{KeyValue: recorder, failGet: func(key string) bool {
-		return key == kvutil.AccountKey(testAccountID, "lt-0unreadable000000")
+		return key == kvstore.AccountKey(testAccountID, "lt-0unreadable000000")
 	}}
 	scan, err := svc.ScanAccount(context.Background(), testAccountID)
 	require.NoError(t, err)

spinifex/domains/ec2/launchtemplate/account_termination_current_behaviour_test.go

@@ -16,7 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -159,7 +159,7 @@ func TestCurrentBehaviour_AccountTermination_LaunchTemplateStorageLayout(t *test
 
 	header := testAccountID + "." + id
 	name := testAccountID + ".name.776562283129" // hex of "web(1)"
-	assert.Equal(t, sortedKeys(header, header+".v1", header+".v2", name, kvutil.VersionKey), storedKeys(t, svc.kv))
+	assert.Equal(t, sortedKeys(header, header+".v1", header+".v2", name, kvstore.VersionKey), storedKeys(t, svc.kv))
 
 	entry, err := svc.kv.Get(t.Context(), name)
 	require.NoError(t, err)
@@ -223,13 +223,13 @@ func TestAccountTermination_DeleteLaunchTemplateKeepsTheTemplateWhenAStepFails(t
 			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 
 			svc.kv = inner
-			assert.Equal(t, sortedKeys(append(tc.left(header, nameIdx), kvutil.VersionKey)...), storedKeys(t, inner))
+			assert.Equal(t, sortedKeys(append(tc.left(header, nameIdx), kvstore.VersionKey)...), storedKeys(t, inner))
 			assert.Equal(t, []string{id}, describedTemplateIDs(t, svc, testAccountID))
 
 			out, err := svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)
 			require.NoError(t, err)
 			assert.Equal(t, id, aws.StringValue(out.LaunchTemplate.LaunchTemplateId))
-			assert.Equal(t, []string{kvutil.VersionKey}, storedKeys(t, inner))
+			assert.Equal(t, []string{kvstore.VersionKey}, storedKeys(t, inner))
 			assert.Empty(t, describedTemplateIDs(t, svc, testAccountID))
 		})
 	}
@@ -260,7 +260,7 @@ func TestAccountTermination_DeleteLaunchTemplateRestartsWhenTheHeaderChanges(t *
 	_, err = svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)
 	require.NoError(t, err)
 	assert.True(t, raced)
-	assert.Equal(t, []string{kvutil.VersionKey}, storedKeys(t, inner))
+	assert.Equal(t, []string{kvstore.VersionKey}, storedKeys(t, inner))
 }
 
 // A create that fails after claiming the name removes what it wrote, so no
@@ -287,7 +287,7 @@ func TestAccountTermination_FailedCreateRemovesWhatItWrote(t *testing.T) {
 			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 
 			svc.kv = inner
-			assert.Equal(t, []string{kvutil.VersionKey}, storedKeys(t, inner))
+			assert.Equal(t, []string{kvstore.VersionKey}, storedKeys(t, inner))
 			retried := createTemplate(t, svc, "web", "t3.micro")
 			assert.Equal(t, []string{aws.StringValue(retried.LaunchTemplateId)}, describedTemplateIDs(t, svc, testAccountID))
 		})
@@ -299,7 +299,7 @@ func TestAccountTermination_FailedCreateRemovesWhatItWrote(t *testing.T) {
 // failure is an internal error, never a missing template.
 func TestCurrentBehaviour_AccountTermination_DescribeLaunchTemplatesHidesUnreadableHeaders(t *testing.T) {
 	const brokenID = "lt-0badbadbadbadbad0"
-	brokenHeader := kvutil.AccountKey(testAccountID, brokenID)
+	brokenHeader := kvstore.AccountKey(testAccountID, brokenID)
 	tests := []struct {
 		name       string
 		seed       func(t *testing.T, svc *LaunchTemplateServiceImpl)
@@ -350,7 +350,7 @@ func TestAccountCleanup_KeepsTemplatesAndRemovesSettledResidue(t *testing.T) {
 	require.NoError(t, err)
 	orphanBody := versionKey(testAccountID, "lt-0orphan000000000", 1)
 	orphanName := nameKey(testAccountID, "gone")
-	brokenHeader := kvutil.AccountKey(testAccountID, "lt-0broken000000000")
+	brokenHeader := kvstore.AccountKey(testAccountID, "lt-0broken000000000")
 	brokenBody := versionKey(testAccountID, "lt-0broken000000000", 1)
 	brokenName := nameKey(testAccountID, "broken")
 	for key, value := range map[string]string{
@@ -384,7 +384,7 @@ func TestAccountCleanup_KeepsTemplatesAndRemovesSettledResidue(t *testing.T) {
 func TestAccountCleanup_InterruptedTemplateDeleteIsLeftToThePublicRetry(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	kv := svc.kv
 	svc.kv = &faultyKV{KeyValue: kv, failDelete: func(key string) bool { return key == header }}
 	_, err := svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)

spinifex/domains/ec2/launchtemplate/service_impl.go

@@ -19,7 +19,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -64,7 +64,7 @@ func NewLaunchTemplateServiceImplWithNATS(ctx context.Context, cfg *config.Confi
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketLaunchTemplates, 10)
+	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketLaunchTemplates, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketLaunchTemplates, err)
 	}
@@ -87,24 +87,24 @@ func NewLaunchTemplateServiceImplWithNATS(ctx context.Context, cfg *config.Confi
 // Launch template names allow characters such as parentheses that are not valid
 // in NATS KV keys, so the name must not be used in its raw form here.
 func nameKey(accountID, name string) string {
-	return kvutil.AccountKey(accountID, "name."+hex.EncodeToString([]byte(name)))
+	return kvstore.AccountKey(accountID, "name."+hex.EncodeToString([]byte(name)))
 }
 
 // versionKey returns a version-body key: account.lt-<id>.v<n>.
 func versionKey(accountID, ltID string, n int64) string {
-	return kvutil.AccountKey(accountID, ltID+".v"+strconv.FormatInt(n, 10))
+	return kvstore.AccountKey(accountID, ltID+".v"+strconv.FormatInt(n, 10))
 }
 
 // versionPrefix returns the scan prefix for a template's version bodies.
 func versionPrefix(accountID, ltID string) string {
-	return kvutil.AccountKey(accountID, ltID+".v")
+	return kvstore.AccountKey(accountID, ltID+".v")
 }
 
 // --- record load helpers ---
 
 // getHeaderByID reads a header and its KV entry (for CAS) by launch template id.
 func (s *LaunchTemplateServiceImpl) getHeaderByID(ctx context.Context, accountID, ltID string) (*LaunchTemplateHeader, jetstream.KeyValueEntry, error) {
-	entry, err := s.kv.Get(ctx, kvutil.AccountKey(accountID, ltID))
+	entry, err := s.kv.Get(ctx, kvstore.AccountKey(accountID, ltID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
 			return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdNotFound)
@@ -386,7 +386,7 @@ func (s *LaunchTemplateServiceImpl) claimName(ctx context.Context, accountID, na
 	// header means the name is taken; any other read error (transient fault, or a
 	// concurrent in-flight create whose header is not yet written) fails closed so
 	// the name is never stolen from a possibly-live template.
-	_, herr := s.kv.Get(ctx, kvutil.AccountKey(accountID, string(entry.Value())))
+	_, herr := s.kv.Get(ctx, kvstore.AccountKey(accountID, string(entry.Value())))
 	switch {
 	case herr == nil:
 		return errNameInUse()
@@ -404,7 +404,7 @@ func (s *LaunchTemplateServiceImpl) putHeader(ctx context.Context, accountID str
 	if err != nil {
 		return errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.kv.Put(ctx, kvutil.AccountKey(accountID, h.LaunchTemplateId), data); err != nil {
+	if _, err := s.kv.Put(ctx, kvstore.AccountKey(accountID, h.LaunchTemplateId), data); err != nil {
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 	return nil
@@ -510,13 +510,13 @@ func (s *LaunchTemplateServiceImpl) commitVersion(ctx context.Context, accountID
 // re-reading on a conflict. A non-nil check vets each re-read header and
 // stops the commit with its error.
 func (s *LaunchTemplateServiceImpl) commitHeader(ctx context.Context, op, accountID, ltID string, entry jetstream.KeyValueEntry, check func(*LaunchTemplateHeader) error) error {
-	key := kvutil.AccountKey(accountID, ltID)
+	key := kvstore.AccountKey(accountID, ltID)
 	for range maxVersionRetries {
 		_, err := s.kv.Update(ctx, key, entry.Value(), entry.Revision())
 		if err == nil {
 			return nil
 		}
-		if !kvutil.IsRevisionConflict(err) {
+		if !kvstore.IsRevisionConflict(err) {
 			slog.ErrorContext(ctx, op+": header commit failed", "launchTemplateId", ltID, "err", err)
 			return err
 		}
@@ -590,7 +590,7 @@ func (s *LaunchTemplateServiceImpl) ModifyLaunchTemplate(ctx context.Context, in
 // header deleted meanwhile is NotFound for the identifier the request used.
 func (s *LaunchTemplateServiceImpl) setDefaultVersion(ctx context.Context, accountID string, input *ec2.ModifyLaunchTemplateInput, header *LaunchTemplateHeader, entry jetstream.KeyValueEntry, sel string) (*LaunchTemplateHeader, error) {
 	notFound := fmt.Sprintf("The launch template version %s is not found for the specified launch template.", sel)
-	key := kvutil.AccountKey(accountID, header.LaunchTemplateId)
+	key := kvstore.AccountKey(accountID, header.LaunchTemplateId)
 	for range maxVersionRetries {
 		n, err := s.resolveVersionNumber(ctx, accountID, header, sel)
 		if err != nil {
@@ -611,7 +611,7 @@ func (s *LaunchTemplateServiceImpl) setDefaultVersion(ctx context.Context, accou
 		if err == nil {
 			return header, nil
 		}
-		if !kvutil.IsRevisionConflict(err) {
+		if !kvstore.IsRevisionConflict(err) {
 			slog.ErrorContext(ctx, "ModifyLaunchTemplate: header update failed", "launchTemplateId", header.LaunchTemplateId, "err", err)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
@@ -662,7 +662,7 @@ const maxHeaderDeleteAttempts = 3
 // through the public delete; a header changed in between restarts the sweep.
 func (s *LaunchTemplateServiceImpl) deleteTemplate(ctx context.Context, accountID string, header *LaunchTemplateHeader, entry jetstream.KeyValueEntry) error {
 	ltID := header.LaunchTemplateId
-	headerKey := kvutil.AccountKey(accountID, ltID)
+	headerKey := kvstore.AccountKey(accountID, ltID)
 	for attempt := 1; ; attempt++ {
 		if err := s.deleteVersionBodies(ctx, accountID, ltID); err != nil {
 			return err
@@ -824,7 +824,7 @@ func (s *LaunchTemplateServiceImpl) commitVersionDeletes(ctx context.Context, ac
 	}
 	// A template delete that removed the header before the restore has already
 	// swept its bodies, so the restored ones would be left headerless.
-	if _, gerr := s.kv.Get(ctx, kvutil.AccountKey(accountID, ltID)); errors.Is(gerr, jetstream.ErrKeyNotFound) {
+	if _, gerr := s.kv.Get(ctx, kvstore.AccountKey(accountID, ltID)); errors.Is(gerr, jetstream.ErrKeyNotFound) {
 		for n := range deleted {
 			if derr := s.kv.Delete(ctx, versionKey(accountID, ltID, n)); derr != nil {
 				slog.ErrorContext(ctx, "DeleteLaunchTemplateVersions: failed to remove a restored version", "launchTemplateId", ltID, "version", n, "err", derr)

spinifex/domains/ec2/launchtemplate/service_impl_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -116,7 +116,7 @@ func TestCreateLaunchTemplate_OrphanNameReclaim(t *testing.T) {
 	lt := createTemplate(t, svc, "orphan", "t3.micro")
 
 	// Simulate a crash after the name claim but with the header lost.
-	require.NoError(t, svc.kv.Delete(t.Context(), kvutil.AccountKey(testAccountID, aws.StringValue(lt.LaunchTemplateId))))
+	require.NoError(t, svc.kv.Delete(t.Context(), kvstore.AccountKey(testAccountID, aws.StringValue(lt.LaunchTemplateId))))
 
 	// The name index still points at the now-orphaned id; create must reclaim it.
 	out, err := svc.CreateLaunchTemplate(context.Background(), &ec2.CreateLaunchTemplateInput{

spinifex/domains/ec2/launchtemplate/template_lifecycle_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -38,7 +38,7 @@ func TestAccountTermination_VersionCreateRacingADeleteLeavesNoBody(t *testing.T)
 		t.Run(pause, func(t *testing.T) {
 			svc := setupTestService(t)
 			id := createWithUserData(t, svc, "web")
-			header := kvutil.AccountKey(testAccountID, id)
+			header := kvstore.AccountKey(testAccountID, id)
 			raced := false
 			race := func(key string, at string) {
 				if raced || pause != at {
@@ -71,7 +71,7 @@ func TestAccountTermination_DeleteRacingACommittedVersionRemovesIt(t *testing.T)
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeDelete: func(key string) {
 		if key != header || raced {
@@ -95,7 +95,7 @@ func TestAccountTermination_FailedVersionCommitRemovesTheBody(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	before := accountKeys(t, svc.kv, testAccountID)
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	svc.kv = &faultyKV{KeyValue: svc.kv, failUpdate: func(key string) bool { return key == header }}
 
 	_, err := svc.CreateLaunchTemplateVersion(context.Background(), newVersionInput(id), testAccountID)
@@ -110,7 +110,7 @@ func TestAccountTermination_VersionCommitKeepsAConcurrentHeaderChange(t *testing
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header || raced {
@@ -215,10 +215,10 @@ func TestAccountTemplateReferences_FailsClosed(t *testing.T) {
 			require.NoError(t, err)
 		},
 		"header unreadable": func(t *testing.T, svc *LaunchTemplateServiceImpl, id string) {
-			svc.kv = &faultyKV{KeyValue: svc.kv, failGet: func(key string) bool { return key == kvutil.AccountKey(testAccountID, id) }}
+			svc.kv = &faultyKV{KeyValue: svc.kv, failGet: func(key string) bool { return key == kvstore.AccountKey(testAccountID, id) }}
 		},
 		"header undecodable": func(t *testing.T, svc *LaunchTemplateServiceImpl, _ string) {
-			_, err := svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "lt-0broken000000000"), []byte("{not json"))
+			_, err := svc.kv.Put(t.Context(), kvstore.AccountKey(testAccountID, "lt-0broken000000000"), []byte("{not json"))
 			require.NoError(t, err)
 		},
 	}
@@ -241,7 +241,7 @@ func TestModifyLaunchTemplate_RacingADeleteIsNotFound(t *testing.T) {
 		t.Run(map[bool]string{false: "by-id", true: "by-name"}[byName], func(t *testing.T) {
 			svc := setupTestService(t)
 			id := createWithUserData(t, svc, "web")
-			header := kvutil.AccountKey(testAccountID, id)
+			header := kvstore.AccountKey(testAccountID, id)
 			raced := false
 			svc.kv = &faultyKV{KeyValue: svc.kv, beforeUpdate: func(key string) {
 				if key == header && !raced {
@@ -275,7 +275,7 @@ func TestModifyLaunchTemplate_RetriesAConcurrentHeaderChange(t *testing.T) {
 	_, err := svc.CreateLaunchTemplateVersion(context.Background(), newVersionInput(id), testAccountID) // v3
 	require.NoError(t, err)
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header || raced {
@@ -306,7 +306,7 @@ func TestModifyLaunchTemplate_ConflictRetriesAreBounded(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	attempts := 0
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header {
@@ -333,7 +333,7 @@ func TestModifyLaunchTemplate_ConflictReReadFailureIsInternal(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	conflicted := false
 	svc.kv = &faultyKV{
 		KeyValue: inner,
@@ -410,7 +410,7 @@ func TestDefaultChangeRacingAVersionDeleteFailsNotFound(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header || raced {
@@ -460,7 +460,7 @@ func TestFailedVersionDeleteCommitRestoresTheBodies(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	before := accountKeys(t, svc.kv, testAccountID)
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	svc.kv = &faultyKV{KeyValue: svc.kv, failUpdate: func(key string) bool { return key == header }}
 
 	_, err := deleteVersions(svc, id, "2")
@@ -474,7 +474,7 @@ func TestFailedVersionDeleteCommitRestoresTheBodies(t *testing.T) {
 func TestVersionDeleteRacingATemplateDeleteLeavesNothing(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: svc.kv, beforeUpdate: func(key string) {
 		if key != header || raced {
@@ -531,7 +531,7 @@ func TestDefaultChangeRetriesAfterAConcurrentVersionCreate(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header || raced {

spinifex/domains/ec2/launchtemplate/template_references.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -27,7 +27,7 @@ type TemplateReferences struct {
 // header or body it cannot read or decode, is an error, because the unread
 // record may name the resource. Headerless bodies resolve from no launch.
 func (s *LaunchTemplateServiceImpl) AccountTemplateReferences(ctx context.Context, accountID string) (*TemplateReferences, error) {
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list launch-template keys: %w", err)
 	}

spinifex/domains/ec2/natgw/lifecycle_invariants_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	ec2eip "github.com/mulgadc/spinifex/spinifex/domains/ec2/eip"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -42,7 +42,7 @@ func TestRLC3_NatGatewayDeletesWhileRouted(t *testing.T) {
 		natgwID := createTestNatGateway(t, svc)
 
 		testutil.SeedKV(t, js, kvBucketRouteTables, map[string][]byte{
-			kvutil.AccountKey(testAccountID, "rtb-routed"): fmt.Appendf(nil,
+			kvstore.AccountKey(testAccountID, "rtb-routed"): fmt.Appendf(nil,
 				`{"route_table_id":"rtb-routed","vpc_id":"vpc-test1","routes":[{"destination_cidr_block":"0.0.0.0/0","nat_gateway_id":%q}]}`, natgwID),
 		})
 
@@ -93,7 +93,7 @@ func createTestNatGateway(t *testing.T, svc *NatGatewayServiceImpl) string {
 
 func readEIP(t *testing.T, svc *NatGatewayServiceImpl, allocID string) ec2eip.EIPRecord {
 	t.Helper()
-	entry, err := svc.eipKV.Get(t.Context(), kvutil.AccountKey(testAccountID, allocID))
+	entry, err := svc.eipKV.Get(t.Context(), kvstore.AccountKey(testAccountID, allocID))
 	require.NoError(t, err)
 	var eip ec2eip.EIPRecord
 	require.NoError(t, json.Unmarshal(entry.Value(), &eip))

spinifex/domains/ec2/natgw/service_impl.go

@@ -20,7 +20,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -45,7 +45,7 @@ func NewNatGatewayServiceImplWithNATS(ctx context.Context, natsConn *nats.Conn)
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	natgwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketNatGateways, 10)
+	natgwKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketNatGateways, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketNatGateways, err)
 	}
@@ -61,17 +61,17 @@ func NewNatGatewayServiceImplWithNATS(ctx context.Context, natsConn *nats.Conn)
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketDeletedNatGateways, err)
 	}
 
-	eipKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
+	eipKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get EIP KV bucket: %w", err)
 	}
 
-	subnetKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketSubnets, 10)
+	subnetKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketSubnets, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get subnet KV bucket: %w", err)
 	}
 
-	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
@@ -92,7 +92,7 @@ func NewNatGatewayServiceImplWithNATS(ctx context.Context, natsConn *nats.Conn)
 // for as long as a client may still ask about it.
 func getOrCreateDeletedBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) {
 	// Terraform polls DescribeNatGateways after delete and expects state=deleted.
-	return kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{
+	return kvstore.GetOrCreateBucketWithOptions(ctx, js, kvstore.BucketOptions{
 		Name:        KVBucketDeletedNatGateways,
 		Description: "Deleted NAT Gateways (auto-expire after 1 hour)",
 		History:     1,
@@ -113,7 +113,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 	allocID := *input.AllocationId
 
 	// Validate subnet exists and get its VPC
-	subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, subnetID))
+	subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, subnetID))
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound)
 	}
@@ -123,7 +123,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 	}
 
 	// Validate EIP exists and is not already associated
-	eipEntry, err := s.eipKV.Get(ctx, kvutil.AccountKey(accountID, allocID))
+	eipEntry, err := s.eipKV.Get(ctx, kvstore.AccountKey(accountID, allocID))
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidAllocationIDNotFound)
 	}
@@ -152,7 +152,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.natgwKV.Put(ctx, kvutil.AccountKey(accountID, natgwID), data); err != nil {
+	if _, err := s.natgwKV.Put(ctx, kvstore.AccountKey(accountID, natgwID), data); err != nil {
 		slog.ErrorContext(ctx, "Failed to store NAT Gateway", "natGatewayId", natgwID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
@@ -165,7 +165,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 	eipRecord.VpcId = subnetRecord.VpcId
 	eipRecord.State = "associated"
 	if eipData, err := json.Marshal(eipRecord); err == nil {
-		if _, err := s.eipKV.Update(ctx, kvutil.AccountKey(accountID, allocID), eipData, eipEntry.Revision()); err != nil {
+		if _, err := s.eipKV.Update(ctx, kvstore.AccountKey(accountID, allocID), eipData, eipEntry.Revision()); err != nil {
 			slog.WarnContext(ctx, "CreateNatGateway: failed to mark EIP associated", "natGatewayId", natgwID, "allocationId", allocID, "err", err)
 		}
 	}
@@ -185,7 +185,7 @@ func (s *NatGatewayServiceImpl) DeleteNatGateway(ctx context.Context, input *ec2
 	}
 
 	natgwID := *input.NatGatewayId
-	key := kvutil.AccountKey(accountID, natgwID)
+	key := kvstore.AccountKey(accountID, natgwID)
 
 	entry, err := s.natgwKV.Get(ctx, key)
 	if err != nil {
@@ -255,7 +255,7 @@ func (s *NatGatewayServiceImpl) publishDeleteEventsForNatGateway(ctx context.Con
 		slog.WarnContext(ctx, "DeleteNatGateway: JetStream setup failed, SNAT teardown deferred to route delete", "natGatewayId", record.NatGatewayId, "err", err)
 		return
 	}
-	rtbKV, err := kvutil.GetOrCreateBucket(ctx, js, kvBucketRouteTables, 10)
+	rtbKV, err := kvstore.GetOrCreateBucket(ctx, js, kvBucketRouteTables, 10)
 	if err != nil {
 		slog.WarnContext(ctx, "DeleteNatGateway: route-table scan failed, SNAT teardown deferred to route delete", "natGatewayId", record.NatGatewayId, "err", err)
 		return
@@ -266,7 +266,7 @@ func (s *NatGatewayServiceImpl) publishDeleteEventsForNatGateway(ctx context.Con
 	}
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := rtbKV.Get(ctx, key)
@@ -302,7 +302,7 @@ func (s *NatGatewayServiceImpl) publishDeleteEventsForNatGateway(ctx context.Con
 			if assoc.SubnetId == "" {
 				continue
 			}
-			subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, assoc.SubnetId))
+			subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, assoc.SubnetId))
 			if err != nil {
 				continue
 			}
@@ -322,7 +322,7 @@ func (s *NatGatewayServiceImpl) disassociateEIP(ctx context.Context, record *Nat
 	if record.AllocationId == "" {
 		return
 	}
-	key := kvutil.AccountKey(accountID, record.AllocationId)
+	key := kvstore.AccountKey(accountID, record.AllocationId)
 	entry, err := s.eipKV.Get(ctx, key)
 	if err != nil {
 		if !errors.Is(err, jetstream.ErrKeyNotFound) {
@@ -380,7 +380,7 @@ func (s *NatGatewayServiceImpl) DescribeNatGateways(ctx context.Context, input *
 	foundIDs := make(map[string]bool)
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 
@@ -415,7 +415,7 @@ func (s *NatGatewayServiceImpl) DescribeNatGateways(ctx context.Context, input *
 		if foundIDs[id] {
 			continue
 		}
-		key := kvutil.AccountKey(accountID, id)
+		key := kvstore.AccountKey(accountID, id)
 		entry, err := s.deletedNatgwKV.Get(ctx, key)
 		if err != nil {
 			if errors.Is(err, jetstream.ErrKeyNotFound) {

spinifex/domains/ec2/natgw/service_impl_test.go

@@ -11,7 +11,7 @@ import (
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -31,18 +31,18 @@ func setupTestServiceJS(t *testing.T) (*NatGatewayServiceImpl, jetstream.JetStre
 
 	// Seed VPC KV
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`),
+		kvstore.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`),
 	})
 
 	// Seed subnet KV (public subnet)
 	testutil.SeedKV(t, js, ec2vpc.KVBucketSubnets, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "subnet-pub1"): []byte(`{"subnet_id":"subnet-pub1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available","map_public_ip_on_launch":true}`),
+		kvstore.AccountKey(testAccountID, "subnet-pub1"): []byte(`{"subnet_id":"subnet-pub1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available","map_public_ip_on_launch":true}`),
 	})
 
 	// Seed EIP KV
 	testutil.SeedKV(t, js, ec2eip.KVBucketEIPs, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "eipalloc-test1"): []byte(`{"allocation_id":"eipalloc-test1","public_ip":"203.0.113.50","state":"allocated"}`),
-		kvutil.AccountKey(testAccountID, "eipalloc-used"):  []byte(`{"allocation_id":"eipalloc-used","public_ip":"203.0.113.51","state":"associated","association_id":"eipassoc-xxx"}`),
+		kvstore.AccountKey(testAccountID, "eipalloc-test1"): []byte(`{"allocation_id":"eipalloc-test1","public_ip":"203.0.113.50","state":"allocated"}`),
+		kvstore.AccountKey(testAccountID, "eipalloc-used"):  []byte(`{"allocation_id":"eipalloc-used","public_ip":"203.0.113.51","state":"associated","association_id":"eipassoc-xxx"}`),
 	})
 
 	svc, err := NewNatGatewayServiceImplWithNATS(t.Context(), nc)
@@ -170,7 +170,7 @@ func TestDeleteNatGateway(t *testing.T) {
 	assert.Equal(t, natgwID, *deleteOut.NatGatewayId)
 
 	// Should be gone
-	_, err = svc.natgwKV.Get(t.Context(), kvutil.AccountKey(testAccountID, natgwID))
+	_, err = svc.natgwKV.Get(t.Context(), kvstore.AccountKey(testAccountID, natgwID))
 	assert.ErrorIs(t, err, jetstream.ErrKeyNotFound)
 }
 

spinifex/domains/ec2/placementgroup/account_cleanup.go

@@ -8,7 +8,7 @@ import (
 	"log/slog"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -39,7 +39,7 @@ type AccountCleanup struct {
 // referenced returns a reason is kept; unreadable records are left for the owner scan to report. A
 // listing failure is returned and nothing is deleted.
 func (s *PlacementGroupServiceImpl) CleanupAccount(ctx context.Context, accountID string, referenced func(groupID, groupName string) string) (*AccountCleanup, error) {
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list placement-group keys: %w", err)
 	}
@@ -47,7 +47,7 @@ func (s *PlacementGroupServiceImpl) CleanupAccount(ctx context.Context, accountI
 	prefix := accountID + "."
 	out := &AccountCleanup{Referenced: map[string]string{}}
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.kv.Get(ctx, key)

spinifex/domains/ec2/placementgroup/account_scan.go

@@ -8,7 +8,7 @@ import (
 	"log/slog"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -34,7 +34,7 @@ type AccountScan struct {
 // A listing failure is returned as an error; per-record read failures are
 // counted. The bucket's schema-version key belongs to no account.
 func (s *PlacementGroupServiceImpl) ScanAccount(ctx context.Context, accountID string) (*AccountScan, error) {
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list placement-group keys: %w", err)
 	}
@@ -42,7 +42,7 @@ func (s *PlacementGroupServiceImpl) ScanAccount(ctx context.Context, accountID s
 	prefix := accountID + "."
 	scan := &AccountScan{}
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.kv.Get(ctx, key)

spinifex/domains/ec2/placementgroup/account_scan_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit/readonlystore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -42,14 +42,14 @@ func TestScanAccountClassifiesPlacementGroupRecords(t *testing.T) {
 		GroupName: "held", NodeInstances: map[string][]string{"node-2": {"i-0bbb", "i-0aaa"}, "node-3": {"i-0aaa"}},
 	}, testAccountID)
 	require.NoError(t, err)
-	_, err = svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "undecodable"), []byte("{not json"))
+	_, err = svc.kv.Put(t.Context(), kvstore.AccountKey(testAccountID, "undecodable"), []byte("{not json"))
 	require.NoError(t, err)
-	_, err = svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "anonymous"), []byte(`{"group_name":"anonymous"}`))
+	_, err = svc.kv.Put(t.Context(), kvstore.AccountKey(testAccountID, "anonymous"), []byte(`{"group_name":"anonymous"}`))
 	require.NoError(t, err)
 	createTestGroup(t, svc, "unreadable", ec2.PlacementStrategySpread)
-	require.Contains(t, bucketKeys(t, svc.kv), kvutil.VersionKey)
+	require.Contains(t, bucketKeys(t, svc.kv), kvstore.VersionKey)
 
-	svc.kv = &hookedKV{KeyValue: readonlystore.NewKV(t, svc.kv), failGet: kvutil.AccountKey(testAccountID, "unreadable")}
+	svc.kv = &hookedKV{KeyValue: readonlystore.NewKV(t, svc.kv), failGet: kvstore.AccountKey(testAccountID, "unreadable")}
 	scan, err := svc.ScanAccount(context.Background(), testAccountID)
 	require.NoError(t, err)
 

spinifex/domains/ec2/placementgroup/account_termination_current_behaviour_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -84,7 +84,7 @@ func TestCurrentBehaviour_AccountTermination_PlacementGroupStorageLayout(t *test
 	}, testAccountID)
 	require.NoError(t, err)
 
-	assert.Equal(t, []string{testAccountID + ".spread-a", kvutil.VersionKey}, bucketKeys(t, svc.kv))
+	assert.Equal(t, []string{testAccountID + ".spread-a", kvstore.VersionKey}, bucketKeys(t, svc.kv))
 
 	entry, err := svc.kv.Get(t.Context(), testAccountID+".spread-a")
 	require.NoError(t, err)
@@ -100,7 +100,7 @@ func TestCurrentBehaviour_AccountTermination_PlacementGroupStorageLayout(t *test
 // a record cannot be removed through DeletePlacementGroup either, and a read
 // failure is an internal error, never an unknown group.
 func TestCurrentBehaviour_AccountTermination_DescribePlacementGroupsHidesUnreadableRecords(t *testing.T) {
-	brokenKey := kvutil.AccountKey(testAccountID, "broken")
+	brokenKey := kvstore.AccountKey(testAccountID, "broken")
 	tests := []struct {
 		name       string
 		seed       func(t *testing.T, svc *PlacementGroupServiceImpl)
@@ -139,7 +139,7 @@ func TestCurrentBehaviour_AccountTermination_DescribePlacementGroupsHidesUnreada
 func TestAccountTermination_DeletePlacementGroupLosesToAConcurrentMember(t *testing.T) {
 	svc := setupTestService(t)
 	createTestGroup(t, svc, "cluster-a", ec2.PlacementStrategyCluster)
-	key := kvutil.AccountKey(testAccountID, "cluster-a")
+	key := kvstore.AccountKey(testAccountID, "cluster-a")
 
 	launcher := &PlacementGroupServiceImpl{kv: svc.kv, region: svc.region}
 	var reserveErr, finalizeErr error
@@ -222,7 +222,7 @@ func TestAccountCleanup_DeletesEmptyGroupsAndBlocksOnMembers(t *testing.T) {
 	empty := createTestGroup(t, svc, "empty", ec2.PlacementStrategySpread)
 	held := createTestGroup(t, svc, "held", ec2.PlacementStrategyCluster)
 	require.NoError(t, addMember(t, svc, "held", "i-0123456789abcdef0"))
-	_, err := svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "broken"), []byte("{not json"))
+	_, err := svc.kv.Put(t.Context(), kvstore.AccountKey(testAccountID, "broken"), []byte("{not json"))
 	require.NoError(t, err)
 	other := "999988887777"
 	_, err = svc.CreatePlacementGroup(context.Background(), &ec2.CreatePlacementGroupInput{
@@ -238,7 +238,7 @@ func TestAccountCleanup_DeletesEmptyGroupsAndBlocksOnMembers(t *testing.T) {
 
 	assert.Equal(t, []string{"held"}, describedGroupNames(t, svc, testAccountID))
 	assert.Equal(t, []string{"theirs"}, describedGroupNames(t, svc, other))
-	_, err = svc.kv.Get(t.Context(), kvutil.AccountKey(testAccountID, "broken"))
+	_, err = svc.kv.Get(t.Context(), kvstore.AccountKey(testAccountID, "broken"))
 	require.NoError(t, err, "an unreadable record is never deleted")
 }
 
@@ -247,7 +247,7 @@ func TestAccountCleanup_DeletesEmptyGroupsAndBlocksOnMembers(t *testing.T) {
 func TestAccountCleanup_ConcurrentMemberBlocksTheDelete(t *testing.T) {
 	svc := setupTestService(t)
 	group := createTestGroup(t, svc, "cluster-a", ec2.PlacementStrategyCluster)
-	key := kvutil.AccountKey(testAccountID, "cluster-a")
+	key := kvstore.AccountKey(testAccountID, "cluster-a")
 	launcher := &PlacementGroupServiceImpl{kv: svc.kv, region: svc.region}
 	var memberErr error
 	// Read 1 is the cleanup's listing read; read 2 is the delete's check.

spinifex/domains/ec2/placementgroup/service_impl.go

@@ -17,7 +17,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -87,7 +87,7 @@ func NewPlacementGroupServiceImplWithNATS(ctx context.Context, cfg *config.Confi
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketPlacementGroups, 10)
+	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketPlacementGroups, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketPlacementGroups, err)
 	}
@@ -137,7 +137,7 @@ func (s *PlacementGroupServiceImpl) CreatePlacementGroup(ctx context.Context, in
 	}
 
 	groupName := *input.GroupName
-	key := kvutil.AccountKey(accountID, groupName)
+	key := kvstore.AccountKey(accountID, groupName)
 	groupID := awsidentifiers.GenerateResourceID("pg")
 
 	record := PlacementGroupRecord{
@@ -175,7 +175,7 @@ func (s *PlacementGroupServiceImpl) DeletePlacementGroup(ctx context.Context, in
 	}
 
 	groupName := *input.GroupName
-	if _, err := s.deleteGroup(ctx, accountID, kvutil.AccountKey(accountID, groupName)); err != nil {
+	if _, err := s.deleteGroup(ctx, accountID, kvstore.AccountKey(accountID, groupName)); err != nil {
 		if errors.Is(err, errPGAbsent) {
 			// Lower case, unlike DescribePlacementGroups' message for the same code.
 			return nil, awserrors.Errorf(awserrors.ErrorInvalidPlacementGroupUnknown, "The placement group '%s' is unknown.", groupName)
@@ -194,7 +194,7 @@ func (s *PlacementGroupServiceImpl) DeletePlacementGroup(ctx context.Context, in
 // The returned count is the members seen by the last check.
 func (s *PlacementGroupServiceImpl) deleteGroup(ctx context.Context, accountID, key string) (int, error) {
 	members := 0
-	record, err := kvutil.DeleteIf(ctx, s.kv, key, casConfig(), func(record *PlacementGroupRecord) error {
+	record, err := kvstore.DeleteIf(ctx, s.kv, key, casConfig(), func(record *PlacementGroupRecord) error {
 		members = memberCount(record.NodeInstances)
 		if members > 0 {
 			return errPGInUse
@@ -238,7 +238,7 @@ func (s *PlacementGroupServiceImpl) DescribePlacementGroups(ctx context.Context,
 
 	var groups []*ec2.PlacementGroup
 	for _, k := range keys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -341,7 +341,7 @@ func pgMatchesFilters(record *PlacementGroupRecord, filters map[string][]string)
 
 const maxCASRetries = 5
 
-// errPGAbsent and errPGContended mark the two kvutil outcomes that map to
+// errPGAbsent and errPGContended mark the two kvstore outcomes that map to
 // specific AWS errors rather than a generic internal failure, and neither
 // escapes this file. errPGInUse is the AWS error itself.
 var (
@@ -350,8 +350,8 @@ var (
 	errPGInUse     = errors.New(awserrors.ErrorInvalidPlacementGroupInUse)
 )
 
-// mapCASError turns a kvutil failure into the AWS error the caller returns.
-// A non-kvutil error came from mutate and is already an AWS code.
+// mapCASError turns a kvstore failure into the AWS error the caller returns.
+// A non-kvstore error came from mutate and is already an AWS code.
 func mapCASError(ctx context.Context, op, groupName, accountID string, err error) error {
 	switch {
 	case errors.Is(err, errPGAbsent):
@@ -359,16 +359,16 @@ func mapCASError(ctx context.Context, op, groupName, accountID string, err error
 	case errors.Is(err, errPGContended):
 		slog.ErrorContext(ctx, "Placement group CAS retries exhausted under contention",
 			"op", op, "groupName", groupName, "accountID", accountID)
-	case errors.Is(err, kvutil.ErrRead):
+	case errors.Is(err, kvstore.ErrRead):
 		slog.ErrorContext(ctx, "Failed to read placement group from KV",
 			"op", op, "groupName", groupName, "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrDecode):
+	case errors.Is(err, kvstore.ErrDecode):
 		slog.ErrorContext(ctx, "Corrupt placement group record in KV",
 			"op", op, "groupName", groupName, "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrEncode):
+	case errors.Is(err, kvstore.ErrEncode):
 		slog.ErrorContext(ctx, "Failed to marshal placement group record",
 			"op", op, "groupName", groupName, "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrWrite):
+	case errors.Is(err, kvstore.ErrWrite):
 		slog.ErrorContext(ctx, "Failed to write placement group to KV",
 			"op", op, "groupName", groupName, "accountID", accountID, "err", err)
 	default:
@@ -378,8 +378,8 @@ func mapCASError(ctx context.Context, op, groupName, accountID string, err error
 }
 
 // casConfig is the shared CAS policy for every placement group mutation.
-func casConfig() kvutil.CASConfig {
-	return kvutil.CASConfig{
+func casConfig() kvstore.CASConfig {
+	return kvstore.CASConfig{
 		Attempts:  maxCASRetries,
 		NotFound:  errPGAbsent,
 		Exhausted: func(string, int) error { return errPGContended },
@@ -394,7 +394,7 @@ func (s *PlacementGroupServiceImpl) ReserveSpreadNodes(ctx context.Context, inpu
 	}
 
 	var selected []string
-	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			if record.State != ec2.PlacementGroupStateAvailable {
 				return false, errors.New(awserrors.ErrorInvalidPlacementGroupUnknown)
@@ -437,7 +437,7 @@ func (s *PlacementGroupServiceImpl) FinalizeSpreadInstances(ctx context.Context,
 		return nil, errors.New(awserrors.ErrorMissingParameter)
 	}
 
-	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			maps.Copy(record.NodeInstances, input.NodeInstances)
 			return true, nil
@@ -462,7 +462,7 @@ func (s *PlacementGroupServiceImpl) ReleaseSpreadNodes(ctx context.Context, inpu
 		releaseSet[n] = true
 	}
 
-	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			changed := false
 			for node := range releaseSet {
@@ -490,7 +490,7 @@ func (s *PlacementGroupServiceImpl) RemoveInstance(ctx context.Context, input *R
 		return nil, errors.New(awserrors.ErrorMissingParameter)
 	}
 
-	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			instances, exists := record.NodeInstances[input.NodeName]
 			if !exists {
@@ -533,7 +533,7 @@ func (s *PlacementGroupServiceImpl) ReserveClusterNode(ctx context.Context, inpu
 	}
 
 	var targetNode string
-	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			if record.State != ec2.PlacementGroupStateAvailable {
 				return false, errors.New(awserrors.ErrorInvalidPlacementGroupUnknown)
@@ -570,7 +570,7 @@ func (s *PlacementGroupServiceImpl) FinalizeClusterInstances(ctx context.Context
 		return nil, errors.New(awserrors.ErrorMissingParameter)
 	}
 
-	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			// Append rather than replace: a cluster group can have several launches in flight at once.
 			for node, ids := range input.NodeInstances {

spinifex/domains/ec2/placementgroup/service_impl_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -39,7 +39,7 @@ func createTestGroup(t *testing.T, svc *PlacementGroupServiceImpl, name, strateg
 // readRecord returns the stored record for assertions.
 func readRecord(t *testing.T, svc *PlacementGroupServiceImpl, groupName string) *PlacementGroupRecord {
 	t.Helper()
-	entry, err := svc.kv.Get(t.Context(), kvutil.AccountKey(testAccountID, groupName))
+	entry, err := svc.kv.Get(t.Context(), kvstore.AccountKey(testAccountID, groupName))
 	require.NoError(t, err)
 	var record PlacementGroupRecord
 	require.NoError(t, json.Unmarshal(entry.Value(), &record))
@@ -50,7 +50,7 @@ func readRecord(t *testing.T, svc *PlacementGroupServiceImpl, groupName string)
 // starting state the public API cannot produce.
 func seedRecord(t *testing.T, svc *PlacementGroupServiceImpl, groupName string, mutate func(*PlacementGroupRecord)) {
 	t.Helper()
-	_, err := kvutil.Update(t.Context(), svc.kv, kvutil.AccountKey(testAccountID, groupName), kvutil.CASConfig{},
+	_, err := kvstore.Update(t.Context(), svc.kv, kvstore.AccountKey(testAccountID, groupName), kvstore.CASConfig{},
 		func(record *PlacementGroupRecord) (bool, error) {
 			mutate(record)
 			return true, nil

spinifex/domains/ec2/routetable/service_impl.go

@@ -25,7 +25,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -52,7 +52,7 @@ func NewRouteTableServiceImplWithNATS(ctx context.Context, cfg *config.Config, n
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	rtbKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketRouteTables, 10)
+	rtbKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketRouteTables, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketRouteTables, err)
 	}
@@ -60,22 +60,22 @@ func NewRouteTableServiceImplWithNATS(ctx context.Context, cfg *config.Config, n
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketRouteTables, err)
 	}
 
-	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
 
-	igwKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2igw.KVBucketIGW, 10)
+	igwKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2igw.KVBucketIGW, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get IGW KV bucket: %w", err)
 	}
 
-	subnetKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketSubnets, 10)
+	subnetKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketSubnets, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get subnet KV bucket: %w", err)
 	}
 
-	natgwKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2natgw.KVBucketNatGateways, 10)
+	natgwKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2natgw.KVBucketNatGateways, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get NAT Gateway KV bucket: %w", err)
 	}
@@ -95,7 +95,7 @@ func NewRouteTableServiceImplWithNATS(ctx context.Context, cfg *config.Config, n
 
 // getRouteTable retrieves a route table record from KV.
 func (s *RouteTableServiceImpl) getRouteTable(ctx context.Context, accountID, rtbID string) (*RouteTableRecord, error) {
-	entry, err := s.rtbKV.Get(ctx, kvutil.AccountKey(accountID, rtbID))
+	entry, err := s.rtbKV.Get(ctx, kvstore.AccountKey(accountID, rtbID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
 			return nil, rtbNotFoundError(rtbID)
@@ -118,7 +118,7 @@ func (s *RouteTableServiceImpl) putRouteTable(ctx context.Context, accountID str
 		slog.ErrorContext(ctx, "Failed to marshal route table record", "routeTableId", record.RouteTableId, "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.rtbKV.Put(ctx, kvutil.AccountKey(accountID, record.RouteTableId), data); err != nil {
+	if _, err := s.rtbKV.Put(ctx, kvstore.AccountKey(accountID, record.RouteTableId), data); err != nil {
 		slog.ErrorContext(ctx, "Failed to write route table to KV", "routeTableId", record.RouteTableId, "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
@@ -130,7 +130,7 @@ func (s *RouteTableServiceImpl) putRouteTable(ctx context.Context, accountID str
 // parallel AssociateRouteTable calls Terraform fans out per route table.
 const rtbCASMaxRetries = 16
 
-// errRTBAbsent and errRTBContended mark the two kvutil outcomes that map to
+// errRTBAbsent and errRTBContended mark the two kvstore outcomes that map to
 // specific AWS errors rather than a generic internal failure. Neither
 // escapes this file.
 var (
@@ -143,7 +143,7 @@ var (
 // several subnets with one route table at once. mutate reports whether
 // it changed the record; a false return commits nothing.
 func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, accountID, rtbID string, mutate func(*RouteTableRecord) (bool, error)) error {
-	_, err := kvutil.Update(ctx, s.rtbKV, kvutil.AccountKey(accountID, rtbID), kvutil.CASConfig{
+	_, err := kvstore.Update(ctx, s.rtbKV, kvstore.AccountKey(accountID, rtbID), kvstore.CASConfig{
 		Attempts:  rtbCASMaxRetries,
 		NotFound:  errRTBAbsent,
 		Exhausted: func(string, int) error { return errRTBContended },
@@ -157,13 +157,13 @@ func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, account
 		// Contended rather than broken.
 		slog.ErrorContext(ctx, "Route table CAS retries exhausted under contention",
 			"routeTableId", rtbID, "attempts", rtbCASMaxRetries)
-	case errors.Is(err, kvutil.ErrRead):
+	case errors.Is(err, kvstore.ErrRead):
 		slog.ErrorContext(ctx, "Failed to read route table from KV", "routeTableId", rtbID, "err", err)
-	case errors.Is(err, kvutil.ErrDecode):
+	case errors.Is(err, kvstore.ErrDecode):
 		slog.ErrorContext(ctx, "Corrupt route table record in KV", "routeTableId", rtbID, "err", err)
-	case errors.Is(err, kvutil.ErrEncode):
+	case errors.Is(err, kvstore.ErrEncode):
 		slog.ErrorContext(ctx, "Failed to marshal route table record", "routeTableId", rtbID, "err", err)
-	case errors.Is(err, kvutil.ErrWrite):
+	case errors.Is(err, kvstore.ErrWrite):
 		slog.ErrorContext(ctx, "Failed to write route table to KV", "routeTableId", rtbID, "err", err)
 	default:
 		return err
@@ -173,7 +173,7 @@ func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, account
 
 // getVPCCidr looks up a VPC's CIDR block from the VPC KV bucket.
 func (s *RouteTableServiceImpl) getVPCCidr(ctx context.Context, accountID, vpcID string) (string, error) {
-	entry, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID))
+	entry, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID))
 	if err != nil {
 		return "", awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
 	}
@@ -229,7 +229,7 @@ func (s *RouteTableServiceImpl) subnetsImplicitlyOnMainRT(ctx context.Context, a
 	}
 	var implicit []string
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.subnetKV.Get(ctx, key)
@@ -387,7 +387,7 @@ func (s *RouteTableServiceImpl) allSubnetsForVPC(ctx context.Context, accountID,
 	}
 	var subnets []string
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.subnetKV.Get(ctx, key)
@@ -455,7 +455,7 @@ func (s *RouteTableServiceImpl) allRouteTablesForVPC(ctx context.Context, accoun
 
 	var results []RouteTableRecord
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.rtbKV.Get(ctx, key)
@@ -577,7 +577,7 @@ func (s *RouteTableServiceImpl) DeleteRouteTable(ctx context.Context, input *ec2
 		}
 	}
 
-	if err := s.rtbKV.Delete(ctx, kvutil.AccountKey(accountID, rtbID)); err != nil {
+	if err := s.rtbKV.Delete(ctx, kvstore.AccountKey(accountID, rtbID)); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -633,7 +633,7 @@ func (s *RouteTableServiceImpl) DescribeRouteTables(ctx context.Context, input *
 	foundIDs := make(map[string]bool)
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 
@@ -740,7 +740,7 @@ func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.Crea
 	case input.GatewayId != nil && *input.GatewayId != "":
 		igwID := *input.GatewayId
 		// Verify IGW exists and is attached to the same VPC
-		igwEntry, err := s.igwKV.Get(ctx, kvutil.AccountKey(accountID, igwID))
+		igwEntry, err := s.igwKV.Get(ctx, kvstore.AccountKey(accountID, igwID))
 		if err != nil {
 			return nil, gatewayNotFoundError(igwID)
 		}
@@ -770,7 +770,7 @@ func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.Crea
 	case input.NatGatewayId != nil && *input.NatGatewayId != "":
 		natgwID := *input.NatGatewayId
 		// Verify NAT GW exists and belongs to the same VPC
-		natgwEntry, err := s.natgwKV.Get(ctx, kvutil.AccountKey(accountID, natgwID))
+		natgwEntry, err := s.natgwKV.Get(ctx, kvstore.AccountKey(accountID, natgwID))
 		if err != nil {
 			return nil, errors.New(awserrors.ErrorInvalidNatGatewayIDNotFound)
 		}
@@ -924,7 +924,7 @@ func (s *RouteTableServiceImpl) ReplaceRoute(ctx context.Context, input *ec2.Rep
 	igwID := *input.GatewayId
 
 	// AWS reports a missing gateway ahead of a missing route.
-	igwEntry, err := s.igwKV.Get(ctx, kvutil.AccountKey(accountID, igwID))
+	igwEntry, err := s.igwKV.Get(ctx, kvstore.AccountKey(accountID, igwID))
 	if err != nil {
 		return nil, gatewayNotFoundError(igwID)
 	}
@@ -983,7 +983,7 @@ func (s *RouteTableServiceImpl) AssociateRouteTable(ctx context.Context, input *
 	}
 
 	// Verify subnet exists and belongs to the same VPC
-	subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, subnetID))
+	subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, subnetID))
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound)
 	}
@@ -1077,7 +1077,7 @@ func (s *RouteTableServiceImpl) DisassociateRouteTable(ctx context.Context, inpu
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 
@@ -1168,7 +1168,7 @@ func (s *RouteTableServiceImpl) ReplaceRouteTableAssociation(ctx context.Context
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 
@@ -1369,7 +1369,7 @@ func (s *RouteTableServiceImpl) publishNatGatewayDeleteEvents(ctx context.Contex
 	if s.natsConn == nil {
 		return
 	}
-	natgwEntry, err := s.natgwKV.Get(ctx, kvutil.AccountKey(accountID, natgwID))
+	natgwEntry, err := s.natgwKV.Get(ctx, kvstore.AccountKey(accountID, natgwID))
 	if err != nil {
 		slog.WarnContext(ctx, "NAT GW event: natgw lookup failed", "topic", networkv1.NATGatewayDeleteSubject, "natGatewayId", natgwID, "err", err)
 		return
@@ -1417,7 +1417,7 @@ func (s *RouteTableServiceImpl) publishNatGatewayEventsForAssociation(ctx contex
 		if r.NatGatewayId == "" {
 			continue
 		}
-		natgwEntry, err := s.natgwKV.Get(ctx, kvutil.AccountKey(accountID, r.NatGatewayId))
+		natgwEntry, err := s.natgwKV.Get(ctx, kvstore.AccountKey(accountID, r.NatGatewayId))
 		if err != nil {
 			slog.WarnContext(ctx, "NAT GW event: natgw lookup failed", "topic", topic, "natGatewayId", r.NatGatewayId, "err", err)
 			continue
@@ -1442,7 +1442,7 @@ func (s *RouteTableServiceImpl) publishNatGatewayEventForSubnet(ctx context.Cont
 	if s.natsConn == nil {
 		return
 	}
-	subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, subnetID))
+	subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, subnetID))
 	if err != nil {
 		slog.WarnContext(ctx, "NAT GW event: subnet lookup failed", "topic", topic, "subnetId", subnetID, "err", err)
 		return

spinifex/domains/ec2/routetable/service_impl_test.go

@@ -13,7 +13,7 @@ import (
 	ec2natgw "github.com/mulgadc/spinifex/spinifex/domains/ec2/natgw"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -28,23 +28,23 @@ func setupTestService(t *testing.T) *RouteTableServiceImpl {
 
 	// Seed VPC KV
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`),
+		kvstore.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`),
 	})
 
 	// Seed IGW KV (attached to vpc-test1)
 	testutil.SeedKV(t, js, ec2igw.KVBucketIGW, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "igw-test1"): []byte(`{"internet_gateway_id":"igw-test1","vpc_id":"vpc-test1","state":"available"}`),
+		kvstore.AccountKey(testAccountID, "igw-test1"): []byte(`{"internet_gateway_id":"igw-test1","vpc_id":"vpc-test1","state":"available"}`),
 	})
 
 	// Seed subnet KV
 	testutil.SeedKV(t, js, ec2vpc.KVBucketSubnets, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "subnet-test1"): []byte(`{"subnet_id":"subnet-test1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available"}`),
-		kvutil.AccountKey(testAccountID, "subnet-priv1"): []byte(`{"subnet_id":"subnet-priv1","vpc_id":"vpc-test1","cidr_block":"10.0.11.0/24","state":"available"}`),
+		kvstore.AccountKey(testAccountID, "subnet-test1"): []byte(`{"subnet_id":"subnet-test1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available"}`),
+		kvstore.AccountKey(testAccountID, "subnet-priv1"): []byte(`{"subnet_id":"subnet-priv1","vpc_id":"vpc-test1","cidr_block":"10.0.11.0/24","state":"available"}`),
 	})
 
 	// Seed NAT Gateway KV
 	testutil.SeedKV(t, js, ec2natgw.KVBucketNatGateways, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "nat-test1"): []byte(`{"nat_gateway_id":"nat-test1","vpc_id":"vpc-test1","subnet_id":"subnet-test1","public_ip":"192.168.1.100","state":"available"}`),
+		kvstore.AccountKey(testAccountID, "nat-test1"): []byte(`{"nat_gateway_id":"nat-test1","vpc_id":"vpc-test1","subnet_id":"subnet-test1","public_ip":"192.168.1.100","state":"available"}`),
 	})
 
 	svc, err := NewRouteTableServiceImplWithNATS(t.Context(), nil, nc)

spinifex/domains/ec2/spotinstance/account_scan.go

@@ -10,7 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/nats-io/nats.go/jetstream"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 )
 
 // AccountSpotRequest is one persisted request in an account scan.
@@ -54,7 +54,7 @@ func scanBucket(ctx context.Context, kv jetstream.KeyValue, accountID string) ([
 	prefix := accountID + "."
 	var out []AccountSpotRequest
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := kv.Get(ctx, key)

spinifex/domains/ec2/spotinstance/account_scan_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 )
 
 func TestScanAccountListsActiveAndTerminalForOneAccount(t *testing.T) {
@@ -35,7 +35,7 @@ func TestScanAccountListsActiveAndTerminalForOneAccount(t *testing.T) {
 func TestScanAccountFailsOnAnUndecodableRecord(t *testing.T) {
 	svc := setupTestService(t)
 	putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"))
-	_, err := svc.activeKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-bad"), []byte("{not json"))
+	_, err := svc.activeKV.Put(t.Context(), kvstore.AccountKey(testAccountID, "sir-bad"), []byte("{not json"))
 	require.NoError(t, err)
 
 	scan, err := svc.ScanAccount(t.Context(), testAccountID)
@@ -45,7 +45,7 @@ func TestScanAccountFailsOnAnUndecodableRecord(t *testing.T) {
 
 func TestScanAccountFailsOnARecordWithoutARequest(t *testing.T) {
 	svc := setupTestService(t)
-	_, err := svc.terminalKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-empty"), []byte(`{"account_id":"x"}`))
+	_, err := svc.terminalKV.Put(t.Context(), kvstore.AccountKey(testAccountID, "sir-empty"), []byte(`{"account_id":"x"}`))
 	require.NoError(t, err)
 
 	_, err = svc.ScanAccount(t.Context(), testAccountID)
@@ -76,7 +76,7 @@ func (k failingGetKV) Get(ctx context.Context, key string) (jetstream.KeyValueEn
 func TestScanAccountFailsWhenARecordCannotBeRead(t *testing.T) {
 	svc := setupTestService(t)
 	putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"), makeSIR("sir-bbb", "i-bbb"))
-	svc.activeKV = failingGetKV{KeyValue: svc.activeKV, key: kvutil.AccountKey(testAccountID, "sir-bbb")}
+	svc.activeKV = failingGetKV{KeyValue: svc.activeKV, key: kvstore.AccountKey(testAccountID, "sir-bbb")}
 
 	scan, err := svc.ScanAccount(t.Context(), testAccountID)
 	require.Error(t, err)

spinifex/domains/ec2/spotinstance/launch_references.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -24,14 +24,14 @@ type LaunchReferences struct {
 // account. It is complete or an error: an unlistable bucket or an unreadable
 // request is an error, because the unread request may name the resource.
 func (s *SpotInstanceServiceImpl) AccountLaunchReferences(ctx context.Context, accountID string) (*LaunchReferences, error) {
-	keys, err := kvutil.Keys(ctx, s.activeKV)
+	keys, err := kvstore.Keys(ctx, s.activeKV)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list spot requests: %w", err)
 	}
 	prefix := accountID + "."
 	out := &LaunchReferences{}
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		record, err := s.readRecord(ctx, s.activeKV, key)

spinifex/domains/ec2/spotinstance/launch_references_test.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -41,7 +41,7 @@ func TestAccountLaunchReferences_ReadsOpenAndActiveRequests(t *testing.T) {
 func TestAccountLaunchReferences_UnreadableRequestFailsTheCheck(t *testing.T) {
 	svc := setupTestService(t)
 	putSIRs(t, svc, makeSIR("sir-active01", "i-0000000000000001"))
-	_, err := svc.activeKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-broken01"), []byte("{not json"))
+	_, err := svc.activeKV.Put(t.Context(), kvstore.AccountKey(testAccountID, "sir-broken01"), []byte("{not json"))
 	require.NoError(t, err)
 
 	_, err = svc.AccountLaunchReferences(t.Context(), testAccountID)

spinifex/domains/ec2/spotinstance/service_impl.go

@@ -15,7 +15,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -41,7 +41,7 @@ func NewSpotInstanceServiceImplWithNATS(ctx context.Context, cfg *config.Config,
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	activeKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketSpotRequests, 10)
+	activeKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketSpotRequests, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSpotRequests, err)
 	}
@@ -71,7 +71,7 @@ func NewSpotInstanceServiceImplWithNATS(ctx context.Context, cfg *config.Config,
 // getOrCreateTerminalBucket opens the terminal bucket without changing its
 // existing replica or placement configuration, creating it only when absent.
 func getOrCreateTerminalBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) {
-	return kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{
+	return kvstore.GetOrCreateBucketWithOptions(ctx, js, kvstore.BucketOptions{
 		Name:        KVBucketSpotRequestsTerminal,
 		Description: "Terminal Spot Instance Requests (auto-expire after 1 hour)",
 		History:     1,
@@ -91,7 +91,7 @@ func (s *SpotInstanceServiceImpl) PutSpotInstanceRequests(ctx context.Context, i
 		if err != nil {
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
-		if _, err := s.activeKV.Put(ctx, kvutil.AccountKey(accountID, sirID), data); err != nil {
+		if _, err := s.activeKV.Put(ctx, kvstore.AccountKey(accountID, sirID), data); err != nil {
 			slog.ErrorContext(ctx, "PutSpotInstanceRequests: KV put failed", "sirId", sirID, "err", err)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
@@ -171,7 +171,7 @@ func (s *SpotInstanceServiceImpl) CancelSpotInstanceRequests(ctx context.Context
 	cancelled := make([]*ec2.CancelledSpotInstanceRequest, 0, len(input.SpotInstanceRequestIds))
 	for _, idPtr := range input.SpotInstanceRequestIds {
 		sirID := aws.StringValue(idPtr)
-		key := kvutil.AccountKey(accountID, sirID)
+		key := kvstore.AccountKey(accountID, sirID)
 
 		record, err := s.readRecord(ctx, s.activeKV, key)
 		if err == nil {
@@ -213,7 +213,7 @@ func (s *SpotInstanceServiceImpl) CloseForInstance(ctx context.Context, instance
 
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		record, err := s.readRecord(ctx, s.activeKV, key)
@@ -251,7 +251,7 @@ func (s *SpotInstanceServiceImpl) listRequests(ctx context.Context, kv jetstream
 	prefix := accountID + "."
 	var requests []*ec2.SpotInstanceRequest
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		record, err := s.readRecord(ctx, kv, key)

spinifex/domains/ec2/systeminstance/launcher.go

@@ -57,7 +57,7 @@ const (
 	BootAMI BootMode = "ami"
 )
 
-// SystemInstanceLauncher launches and terminates system-managed VMs.
+// SystemInstanceLauncher launches, reboots and terminates system-managed VMs.
 // Defined here to avoid a circular import between elbv2/eks and daemon.
 type SystemInstanceLauncher interface {
 	// LaunchSystemInstance creates and starts a system-managed VM, returning its ID and private IP.
@@ -65,6 +65,10 @@ type SystemInstanceLauncher interface {
 
 	// TerminateSystemInstance stops and cleans up a system-managed VM.
 	TerminateSystemInstance(instanceID string) error
+
+	// RebootSystemInstance restarts a system-managed load balancer VM into a
+	// fresh QEMU on its owning node, keeping its ID, ENIs and addresses.
+	RebootSystemInstance(instanceID string) error
 }
 
 // SystemInstanceInput describes a system VM to launch.

spinifex/domains/ec2/tagmirror/tagging_current_behaviour_test.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -39,7 +39,7 @@ func (k *interleavingKV) Get(ctx context.Context, key string) (jetstream.KeyValu
 // its write both survive alongside its own tag.
 func TestCurrentBehaviour_Tagging_RecordMirrorKeepsInterleavedWrites(t *testing.T) {
 	const acct, id = "123456789012", "sg-0123456789abcdef0"
-	key := kvutil.AccountKey(acct, id)
+	key := kvstore.AccountKey(acct, id)
 	seed, err := json.Marshal(tagMirrorRecord{Name: "original", Tags: map[string]string{}})
 	require.NoError(t, err)
 	kv := seedKV(t, "tagmirror-cb", map[string][]byte{key: seed})

spinifex/domains/ec2/tagmirror/tagmirror.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -83,8 +83,8 @@ var errRecordAbsent = errors.New("tagmirror: record absent")
 // applying mut and committing at the read revision, retrying on conflict so a
 // concurrent write is never reverted. An absent resource is skipped.
 func UpdateKVRecordTags[R any](ctx context.Context, kv jetstream.KeyValue, accountID, resourceID string, mut func(*R)) error {
-	key := kvutil.AccountKey(accountID, resourceID)
-	_, err := kvutil.Update(ctx, kv, key, kvutil.CASConfig{NotFound: errRecordAbsent}, func(rec *R) (bool, error) {
+	key := kvstore.AccountKey(accountID, resourceID)
+	_, err := kvstore.Update(ctx, kv, key, kvstore.CASConfig{NotFound: errRecordAbsent}, func(rec *R) (bool, error) {
 		mut(rec)
 		return true, nil
 	})

spinifex/domains/ec2/tagmirror/tagmirror_test.go

@@ -8,7 +8,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -75,15 +75,15 @@ func TestUpdateKVRecordTags(t *testing.T) {
 	rec, err := json.Marshal(&tagMirrorRecord{Name: "r1", Tags: map[string]string{"a": "1"}})
 	require.NoError(t, err)
 	kv := seedKV(t, "tagmirror-test", map[string][]byte{
-		kvutil.AccountKey("acct", "res-1"):   rec,
-		kvutil.AccountKey("acct", "res-bad"): []byte("not-json"),
+		kvstore.AccountKey("acct", "res-1"):   rec,
+		kvstore.AccountKey("acct", "res-bad"): []byte("not-json"),
 	})
 
 	t.Run("mutates and persists", func(t *testing.T) {
 		require.NoError(t, UpdateKVRecordTags(t.Context(), kv, "acct", "res-1", func(r *tagMirrorRecord) {
 			r.Tags["b"] = "2"
 		}))
-		entry, err := kv.Get(t.Context(), kvutil.AccountKey("acct", "res-1"))
+		entry, err := kv.Get(t.Context(), kvstore.AccountKey("acct", "res-1"))
 		require.NoError(t, err)
 		var got tagMirrorRecord
 		require.NoError(t, json.Unmarshal(entry.Value(), &got))
@@ -110,14 +110,14 @@ func TestMirrorKVRecordTags(t *testing.T) {
 	nilTags, err := json.Marshal(&tagMirrorRecord{})
 	require.NoError(t, err)
 	kv := seedKV(t, "tagmirror-mirror-test", map[string][]byte{
-		kvutil.AccountKey("acct", "vol-1"): withTags,
-		kvutil.AccountKey("acct", "vol-2"): nilTags,
+		kvstore.AccountKey("acct", "vol-1"): withTags,
+		kvstore.AccountKey("acct", "vol-2"): nilTags,
 	})
 
 	tagsOf := func(r *tagMirrorRecord) *map[string]string { return &r.Tags }
 
 	readTags := func(t *testing.T, id string) map[string]string {
-		entry, err := kv.Get(t.Context(), kvutil.AccountKey("acct", id))
+		entry, err := kv.Get(t.Context(), kvstore.AccountKey("acct", id))
 		require.NoError(t, err)
 		var got tagMirrorRecord
 		require.NoError(t, json.Unmarshal(entry.Value(), &got))
@@ -150,7 +150,7 @@ func TestMirrorKVRecordTags(t *testing.T) {
 	// must never be touched even though it exists in this bucket.
 	otherRec, err := json.Marshal(&tagMirrorRecord{Tags: map[string]string{"x": "y"}})
 	require.NoError(t, err)
-	_, err = kv.Put(t.Context(), kvutil.AccountKey("acct", "snap-other"), otherRec)
+	_, err = kv.Put(t.Context(), kvstore.AccountKey("acct", "snap-other"), otherRec)
 	require.NoError(t, err)
 	require.NoError(t, MirrorKVRecordTags(t.Context(), kv, "acct", "vol-", resources, tagsOf,
 		MergeTagsMut(&ec2.CreateTagsInput{Tags: []*ec2.Tag{

spinifex/domains/ec2/tags/service_impl.go

@@ -23,7 +23,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -274,7 +274,7 @@ func (s *TagsServiceImpl) CreateTags(ctx context.Context, input *ec2.CreateTagsI
 func (s *TagsServiceImpl) taggedResourceIDs(ctx context.Context, accountID string) ([]string, error) {
 	seen := make(map[string]struct{})
 
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list tag keys: %w", err)
 	}

spinifex/domains/ec2/tags/store.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -46,7 +46,7 @@ func resourceIDFromKVKey(key, accountID string) (string, bool) {
 // GetOrCreateTagsBucket opens the tag bucket, creating it at the cluster's
 // replica count if this is the first node to reach it.
 func GetOrCreateTagsBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) {
-	return kvutil.GetOrCreateBucket(ctx, js, KVBucketTags, kvTagsHistory)
+	return kvstore.GetOrCreateBucket(ctx, js, KVBucketTags, kvTagsHistory)
 }
 
 // readKVTags returns a resource's tags from the bucket, and whether the key
@@ -76,7 +76,7 @@ func (s *TagsServiceImpl) readKVTags(ctx context.Context, accountID, resourceID
 // before the store moved.
 func (s *TagsServiceImpl) mutateTags(ctx context.Context, accountID, resourceID string, mutate func(map[string]string)) error {
 	key := tagsKVKey(accountID, resourceID)
-	_, err := kvutil.Update(ctx, s.kv, key, kvutil.CASConfig{
+	_, err := kvstore.Update(ctx, s.kv, key, kvstore.CASConfig{
 		Attempts:       tagCASAttempts,
 		CreateIfAbsent: true,
 		Exhausted: func(key string, attempts int) error {
@@ -104,7 +104,7 @@ func (s *TagsServiceImpl) PutResourceTags(ctx context.Context, accountID, resour
 	if err != nil {
 		return err
 	}
-	return kvutil.Put(ctx, s.kv, tagsKVKey(accountID, resourceID), kvutil.CASConfig{
+	return kvstore.Put(ctx, s.kv, tagsKVKey(accountID, resourceID), kvstore.CASConfig{
 		Attempts:       tagCASAttempts,
 		CreateIfAbsent: true,
 	}, data)

spinifex/domains/ec2/vpc/default_vpc.go

@@ -12,7 +12,7 @@ import (
 
 	networkv1 "github.com/mulgadc/spinifex/contracts/network/v1"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -105,7 +105,7 @@ func (s *VPCServiceImpl) ensureDefaultVPC(ctx context.Context, accountID string,
 			return claim.info(), nil
 		}
 
-		if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, claim.VpcId)); err == nil {
+		if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, claim.VpcId)); err == nil {
 			return claim.info(), nil
 		} else if !errors.Is(err, jetstream.ErrKeyNotFound) {
 			return nil, fmt.Errorf("read default VPC %s: %w", claim.VpcId, err)
@@ -233,7 +233,7 @@ func oldestAccountRecord[T any](ctx context.Context, kv jetstream.KeyValue, acco
 	var bestID string
 	var bestCreated time.Time
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := kv.Get(ctx, key)
@@ -262,7 +262,7 @@ func oldestAccountRecord[T any](ctx context.Context, kv jetstream.KeyValue, acco
 // marks the claim complete. Records are written with Create, so only the caller
 // that actually wrote one announces it to vpcd.
 func (s *VPCServiceImpl) buildDefaultVPC(ctx context.Context, accountID string, claim *defaultVPCClaim) error {
-	created, err := createRecord(ctx, s.vpcKV, kvutil.AccountKey(accountID, claim.VpcId), VPCRecord{
+	created, err := createRecord(ctx, s.vpcKV, kvstore.AccountKey(accountID, claim.VpcId), VPCRecord{
 		VpcId:              claim.VpcId,
 		CidrBlock:          claim.Cidr,
 		State:              "available",
@@ -284,7 +284,7 @@ func (s *VPCServiceImpl) buildDefaultVPC(ctx context.Context, accountID string,
 	if s.config != nil && s.config.AZ != "" {
 		az = s.config.AZ
 	}
-	created, err = createRecord(ctx, s.subnetKV, kvutil.AccountKey(accountID, claim.SubnetId), SubnetRecord{
+	created, err = createRecord(ctx, s.subnetKV, kvstore.AccountKey(accountID, claim.SubnetId), SubnetRecord{
 		SubnetId:            claim.SubnetId,
 		VpcId:               claim.VpcId,
 		CidrBlock:           claim.SubnetCidr,
@@ -320,7 +320,7 @@ func (s *VPCServiceImpl) buildDefaultVPC(ctx context.Context, accountID string,
 		}
 	}
 
-	if _, err := kvutil.Update(ctx, s.defaultsKV, accountID, kvutil.CASConfig{}, func(c *defaultVPCClaim) (bool, error) {
+	if _, err := kvstore.Update(ctx, s.defaultsKV, accountID, kvstore.CASConfig{}, func(c *defaultVPCClaim) (bool, error) {
 		if c.Complete || c.VpcId != claim.VpcId {
 			return false, nil
 		}

spinifex/domains/ec2/vpc/default_vpc_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
@@ -149,12 +149,12 @@ func TestEnsureDefaultVPC_AdoptsOldestExistingDefault(t *testing.T) {
 	const accountID = "000000000051"
 	created := time.Date(2026, 9, 11, 14, 13, 40, 0, time.UTC)
 	for i, id := range []string{"vpc-0000000000000newer", "vpc-0000000000000older"} {
-		putRecord(t, js, ec2vpc.KVBucketVPCs, kvutil.AccountKey(accountID, id), ec2vpc.VPCRecord{
+		putRecord(t, js, ec2vpc.KVBucketVPCs, kvstore.AccountKey(accountID, id), ec2vpc.VPCRecord{
 			VpcId: id, CidrBlock: ec2vpc.DefaultVPCCidr, IsDefault: true, VNI: int64(200 + i),
 			CreatedAt: created.Add(time.Duration(1-i) * time.Minute),
 		})
 	}
-	putRecord(t, js, ec2vpc.KVBucketSubnets, kvutil.AccountKey(accountID, "subnet-00000000000older"), ec2vpc.SubnetRecord{
+	putRecord(t, js, ec2vpc.KVBucketSubnets, kvstore.AccountKey(accountID, "subnet-00000000000older"), ec2vpc.SubnetRecord{
 		SubnetId: "subnet-00000000000older", VpcId: "vpc-0000000000000older", CidrBlock: ec2vpc.DefaultSubnetCidr,
 		IsDefault: true, CreatedAt: created,
 	})
@@ -183,7 +183,7 @@ func TestEnsureDefaultVPC_FinishesHalfBuiltClaim(t *testing.T) {
 		"route_table_id": "rtb-00000000000half", "internet_gateway_id": "igw-00000000000half",
 		"cidr": ec2vpc.DefaultVPCCidr, "subnet_cidr": ec2vpc.DefaultSubnetCidr, "vni": 4242,
 	})
-	putRecord(t, js, ec2vpc.KVBucketVPCs, kvutil.AccountKey(accountID, "vpc-00000000000000half"), ec2vpc.VPCRecord{
+	putRecord(t, js, ec2vpc.KVBucketVPCs, kvstore.AccountKey(accountID, "vpc-00000000000000half"), ec2vpc.VPCRecord{
 		VpcId: "vpc-00000000000000half", CidrBlock: ec2vpc.DefaultVPCCidr, IsDefault: true, VNI: 4242,
 	})
 

spinifex/domains/ec2/vpc/eni.go

@@ -23,7 +23,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -118,7 +118,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2.
 	subnetId := *input.SubnetId
 
 	// Verify subnet exists and belongs to this account
-	subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, subnetId))
+	subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, subnetId))
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound)
 	}
@@ -205,7 +205,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2.
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal ENI record: %w", err)
 	}
-	if _, err := s.eniKV.Put(ctx, kvutil.AccountKey(accountID, eniId), data); err != nil {
+	if _, err := s.eniKV.Put(ctx, kvstore.AccountKey(accountID, eniId), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -251,7 +251,7 @@ func (s *VPCServiceImpl) ForceDeleteInstanceENI(ctx context.Context, accountID,
 }
 
 func (s *VPCServiceImpl) deleteNetworkInterface(ctx context.Context, eniId, accountID string, force bool) (*ec2.DeleteNetworkInterfaceOutput, error) {
-	key := kvutil.AccountKey(accountID, eniId)
+	key := kvstore.AccountKey(accountID, eniId)
 
 	// Get the ENI record
 	entry, err := s.eniKV.Get(ctx, key)
@@ -346,7 +346,7 @@ func (s *VPCServiceImpl) releaseENISideEffects(ctx context.Context, eniId, accou
 // deletion that did not happen.
 func (s *VPCServiceImpl) DetachAndDeleteENI(ctx context.Context, accountID, eniID string, force bool) (deleted bool, err error) {
 	const maxAttempts = 3
-	key := kvutil.AccountKey(accountID, eniID)
+	key := kvstore.AccountKey(accountID, eniID)
 	var lastErr error
 
 	for range maxAttempts {
@@ -458,7 +458,7 @@ func (s *VPCServiceImpl) ModifyNetworkInterfaceAttribute(ctx context.Context, in
 	}
 
 	eniId := *input.NetworkInterfaceId
-	key := kvutil.AccountKey(accountID, eniId)
+	key := kvstore.AccountKey(accountID, eniId)
 
 	entry, err := s.eniKV.Get(ctx, key)
 	if err != nil {
@@ -569,7 +569,7 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *e
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -707,7 +707,7 @@ func (s *VPCServiceImpl) AttachENIWithOwner(accountID, eniId, instanceId string,
 
 func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instanceId string, deviceIndex int64, instanceOwnerID string) (string, error) {
 	attachmentId := awsidentifiers.GenerateResourceID("eni-attach")
-	err := s.casENI(ctx, kvutil.AccountKey(accountID, eniId), eniId, func(record *ENIRecord) error {
+	err := s.casENI(ctx, kvstore.AccountKey(accountID, eniId), eniId, func(record *ENIRecord) error {
 		if record.Status == "in-use" {
 			return errors.New(awserrors.ErrorInvalidNetworkInterfaceInUse)
 		}
@@ -737,7 +737,7 @@ func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instan
 // DetachENI marks an ENI as detached from an instance (internal use by TerminateInstances).
 // accountID scopes the lookup to the correct KV key.
 func (s *VPCServiceImpl) DetachENI(ctx context.Context, accountID, eniId string) error {
-	key := kvutil.AccountKey(accountID, eniId)
+	key := kvstore.AccountKey(accountID, eniId)
 	entry, err := s.eniKV.Get(ctx, key)
 	if err != nil {
 		return eniNotFoundError(eniId)
@@ -773,7 +773,7 @@ func (s *VPCServiceImpl) GetENIRecord(accountID, eniId string) (ENIRecord, error
 }
 
 func (s *VPCServiceImpl) getENIRecord(ctx context.Context, accountID, eniId string) (ENIRecord, error) {
-	key := kvutil.AccountKey(accountID, eniId)
+	key := kvstore.AccountKey(accountID, eniId)
 	entry, err := s.eniKV.Get(ctx, key)
 	if err != nil {
 		return ENIRecord{}, eniNotFoundError(eniId)
@@ -793,7 +793,7 @@ func (s *VPCServiceImpl) UpdateENI(accountID, eniId string, fn func(*ENIRecord))
 }
 
 func (s *VPCServiceImpl) updateENI(ctx context.Context, accountID, eniId string, fn func(*ENIRecord)) error {
-	return s.casENI(ctx, kvutil.AccountKey(accountID, eniId), eniId, func(r *ENIRecord) error {
+	return s.casENI(ctx, kvstore.AccountKey(accountID, eniId), eniId, func(r *ENIRecord) error {
 		fn(r)
 		return nil
 	})
@@ -876,7 +876,7 @@ func (s *VPCServiceImpl) listInstanceENIs(ctx context.Context, accountID, instan
 	prefix := accountID + "."
 	var out []ENIRecord
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, key)
@@ -938,7 +938,7 @@ func (s *VPCServiceImpl) scanENIs(ctx context.Context, match func(*ENIRecord) bo
 
 	var out []AccountENI
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		accountID, _, found := strings.Cut(key, ".")
@@ -1004,7 +1004,7 @@ func (s *VPCServiceImpl) findENIByAttachment(ctx context.Context, accountID, att
 	}
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, key)
@@ -1028,7 +1028,7 @@ func (s *VPCServiceImpl) UpdateENIPublicIP(accountID, eniId, publicIP, poolName
 }
 
 func (s *VPCServiceImpl) updateENIPublicIP(ctx context.Context, accountID, eniId, publicIP, poolName string) error {
-	key := kvutil.AccountKey(accountID, eniId)
+	key := kvstore.AccountKey(accountID, eniId)
 	entry, err := s.eniKV.Get(ctx, key)
 	if err != nil {
 		return fmt.Errorf("ENI %s not found: %w", eniId, err)
@@ -1063,7 +1063,7 @@ func (s *VPCServiceImpl) fillSGNames(ctx context.Context, accountID string, ids
 			continue
 		}
 		names[id] = ""
-		entry, err := s.sgKV.Get(ctx, kvutil.AccountKey(accountID, id))
+		entry, err := s.sgKV.Get(ctx, kvstore.AccountKey(accountID, id))
 		if err != nil {
 			slog.WarnContext(ctx, "fillSGNames: SG read failed", "groupId", id, "err", err)
 			continue
@@ -1080,7 +1080,7 @@ func (s *VPCServiceImpl) fillSGNames(ctx context.Context, accountID string, ids
 // vpcDNSHostnames reports whether the VPC has enableDnsHostnames on. An
 // unreadable VPC reads as off, so its interfaces go unnamed.
 func (s *VPCServiceImpl) vpcDNSHostnames(ctx context.Context, accountID, vpcID string) bool {
-	entry, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID))
+	entry, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID))
 	if err != nil {
 		slog.WarnContext(ctx, "vpcDNSHostnames: VPC read failed", "vpcId", vpcID, "err", err)
 		return false
@@ -1238,7 +1238,7 @@ func (s *VPCServiceImpl) validateSGAttachment(ctx context.Context, accountID str
 
 	// Each SG must exist in the caller's account and belong to the same VPC.
 	for _, sgId := range sgIds {
-		sgEntry, err := s.sgKV.Get(ctx, kvutil.AccountKey(accountID, sgId))
+		sgEntry, err := s.sgKV.Get(ctx, kvstore.AccountKey(accountID, sgId))
 		if err != nil {
 			return sgNotFoundError(sgId)
 		}

spinifex/domains/ec2/vpc/eni_lease_rebind.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/domains/network/projection"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/topology"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -69,7 +69,7 @@ func (s *VPCServiceImpl) findENIAnyAccount(ctx context.Context, eniID string) (*
 
 	suffix := "." + eniID
 	for _, k := range keys {
-		if k == kvutil.VersionKey || !strings.HasSuffix(k, suffix) {
+		if k == kvstore.VersionKey || !strings.HasSuffix(k, suffix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, k)

spinifex/domains/ec2/vpc/eni_stale_read_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -30,7 +30,7 @@ func newStaleReadENI(t *testing.T) (*VPCServiceImpl, string, func() ENIRecord) {
 	// Reads the stored record directly: the wrapped bucket would serve the
 	// check itself from a lagging replica.
 	stored := func() ENIRecord {
-		entry, err := inner.Get(context.Background(), kvutil.AccountKey(testAccountID, eniID))
+		entry, err := inner.Get(context.Background(), kvstore.AccountKey(testAccountID, eniID))
 		require.NoError(t, err)
 		var rec ENIRecord
 		require.NoError(t, json.Unmarshal(entry.Value(), &rec))
@@ -114,7 +114,7 @@ func TestCASENIGivesUpAfterBoundedConflicts(t *testing.T) {
 	t.Parallel()
 	svc, eniID, _ := newStaleReadENI(t)
 	conflicts := 0
-	err := svc.casENI(t.Context(), kvutil.AccountKey(testAccountID, eniID), eniID, func(*ENIRecord) error {
+	err := svc.casENI(t.Context(), kvstore.AccountKey(testAccountID, eniID), eniID, func(*ENIRecord) error {
 		conflicts++
 		// Move the record under the next compare-and-swap, as a competing writer does.
 		return svc.UpdateENI(testAccountID, eniID, func(r *ENIRecord) { r.Description = "other writer" })

spinifex/domains/ec2/vpc/lease_owner.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -42,7 +42,7 @@ func keyWithSuffixExists(ctx context.Context, kv jetstream.KeyValue, resourceID
 	}
 	suffix := "." + resourceID
 	for _, k := range keys {
-		if k != kvutil.VersionKey && strings.HasSuffix(k, suffix) {
+		if k != kvstore.VersionKey && strings.HasSuffix(k, suffix) {
 			return true, nil
 		}
 	}

spinifex/domains/ec2/vpc/security_group.go

@@ -23,7 +23,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -245,7 +245,7 @@ func (s *VPCServiceImpl) CreateSecurityGroup(ctx context.Context, input *ec2.Cre
 	}
 	sgsInVPC := 0
 	for _, k := range sgKeys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -305,7 +305,7 @@ func (s *VPCServiceImpl) CreateSecurityGroup(ctx context.Context, input *ec2.Cre
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal security group record: %w", err)
 	}
-	if _, err := s.sgKV.Put(ctx, kvutil.AccountKey(accountID, groupId), data); err != nil {
+	if _, err := s.sgKV.Put(ctx, kvstore.AccountKey(accountID, groupId), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -331,7 +331,7 @@ func (s *VPCServiceImpl) DeleteSecurityGroup(ctx context.Context, input *ec2.Del
 	}
 
 	groupId := *input.GroupId
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -380,7 +380,7 @@ func (s *VPCServiceImpl) validateSGRuleReferences(ctx context.Context, accountID
 		if r.SourceSG == "" {
 			continue
 		}
-		entry, err := s.sgKV.Get(ctx, kvutil.AccountKey(accountID, r.SourceSG))
+		entry, err := s.sgKV.Get(ctx, kvstore.AccountKey(accountID, r.SourceSG))
 		if err != nil {
 			return sgNotFoundError(r.SourceSG)
 		}
@@ -406,7 +406,7 @@ func (s *VPCServiceImpl) checkSGDependencies(ctx context.Context, accountID, gro
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range eniKeys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, k)
@@ -435,7 +435,7 @@ func (s *VPCServiceImpl) checkSGDependencies(ctx context.Context, accountID, gro
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range sgKeys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, k)
@@ -561,7 +561,7 @@ func (s *VPCServiceImpl) DescribeSecurityGroups(ctx context.Context, input *ec2.
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -713,7 +713,7 @@ func (s *VPCServiceImpl) GetSecurityGroupsForVpc(ctx context.Context, input *ec2
 
 	groups := []*ec2.SecurityGroupForVpc{}
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, key)
@@ -844,7 +844,7 @@ func (s *VPCServiceImpl) DescribeSecurityGroupRules(ctx context.Context, input *
 	emitted := make(map[string]bool)
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, key)
@@ -945,7 +945,7 @@ func (s *VPCServiceImpl) updateSGRuleTags(ctx context.Context, accountID, ruleID
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, key)
@@ -1056,7 +1056,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupIngress(ctx context.Context, inpu
 	}
 
 	groupId := *input.GroupId
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -1134,7 +1134,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupEgress(ctx context.Context, input
 	}
 
 	groupId := *input.GroupId
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -1212,7 +1212,7 @@ func (s *VPCServiceImpl) RevokeSecurityGroupIngress(ctx context.Context, input *
 	}
 
 	groupId := *input.GroupId
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -1277,7 +1277,7 @@ func (s *VPCServiceImpl) RevokeSecurityGroupEgress(ctx context.Context, input *e
 	}
 
 	groupId := *input.GroupId
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -1397,7 +1397,7 @@ func (s *VPCServiceImpl) updateSGRuleDescriptions(ctx context.Context, accountID
 		return errors.New(awserrors.ErrorInvalidParameterCombination)
 	}
 
-	key := kvutil.AccountKey(accountID, req.groupId)
+	key := kvstore.AccountKey(accountID, req.groupId)
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
 		return sgNotFoundError(req.groupId)
@@ -1581,7 +1581,7 @@ func (s *VPCServiceImpl) ModifySecurityGroupRules(ctx context.Context, input *ec
 	}
 
 	groupId := *input.GroupId
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -2158,7 +2158,7 @@ func (s *VPCServiceImpl) storeDefaultSecurityGroup(ctx context.Context, accountI
 	if err != nil {
 		return nil, false, fmt.Errorf("marshal default security group: %w", err)
 	}
-	if _, err := s.sgKV.Create(ctx, kvutil.AccountKey(accountID, groupId), data); err != nil {
+	if _, err := s.sgKV.Create(ctx, kvstore.AccountKey(accountID, groupId), data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
 			return record, false, nil
 		}
@@ -2182,7 +2182,7 @@ func (s *VPCServiceImpl) announceDefaultSecurityGroup(record *SecurityGroupRecor
 // CannotDelete guard for default SGs. Used by DeleteVpc to cascade-delete the
 // per-VPC default SG. Surfaces vpcd tear-down failures to the caller.
 func (s *VPCServiceImpl) deleteSecurityGroupInternal(ctx context.Context, accountID, groupId string) error {
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
 		return fmt.Errorf("read default security group: %w", err)
@@ -2219,7 +2219,7 @@ func (s *VPCServiceImpl) findDefaultSGForVPC(ctx context.Context, accountID, vpc
 		return "", err
 	}
 	for _, k := range keys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, k)

spinifex/domains/ec2/vpc/security_group_modify_test.go

@@ -15,7 +15,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -62,7 +62,7 @@ func tcpPeerRequest(port int64, peer string) *ec2.SecurityGroupRuleRequest {
 // modify wrote nothing.
 func storedSGRecord(t *testing.T, svc *VPCServiceImpl, sgID string) []byte {
 	t.Helper()
-	entry, err := svc.sgKV.Get(t.Context(), kvutil.AccountKey(testAccountID, sgID))
+	entry, err := svc.sgKV.Get(t.Context(), kvstore.AccountKey(testAccountID, sgID))
 	require.NoError(t, err)
 	return entry.Value()
 }
@@ -689,7 +689,7 @@ func TestRevokeSecurityGroupIngress_InvertedPortRangeStillRevocable(t *testing.T
 	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
 	sgID := createTestSG(t, svc, vpcID, "stale-inverted-ports")
 
-	key := kvutil.AccountKey(testAccountID, sgID)
+	key := kvstore.AccountKey(testAccountID, sgID)
 	var rec SecurityGroupRecord
 	require.NoError(t, json.Unmarshal(storedSGRecord(t, svc, sgID), &rec))
 	rec.IngressRules = append(rec.IngressRules, SGRule{

spinifex/domains/ec2/vpc/security_group_rule_validation_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -35,7 +35,7 @@ func plantIngressRules(t *testing.T, svc *VPCServiceImpl, sgID string, rules ...
 	rec.IngressRules = append(rec.IngressRules, rules...)
 	data, err := json.Marshal(rec)
 	require.NoError(t, err)
-	_, err = svc.sgKV.Put(t.Context(), kvutil.AccountKey(testAccountID, sgID), data)
+	_, err = svc.sgKV.Put(t.Context(), kvstore.AccountKey(testAccountID, sgID), data)
 	require.NoError(t, err)
 }
 

spinifex/domains/ec2/vpc/security_group_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -1524,7 +1524,7 @@ func findDefaultSGInVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) string
 	require.NoError(t, err)
 	var matches []string
 	for _, k := range keys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		entry, err := svc.sgKV.Get(t.Context(), k)
@@ -2122,7 +2122,7 @@ func seedAvailableVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) {
 	rec := VPCRecord{VpcId: vpcID, CidrBlock: "10.0.0.0/16", State: "available"}
 	data, err := json.Marshal(rec)
 	require.NoError(t, err)
-	_, err = svc.vpcKV.Put(t.Context(), kvutil.AccountKey(testAccountID, vpcID), data)
+	_, err = svc.vpcKV.Put(t.Context(), kvstore.AccountKey(testAccountID, vpcID), data)
 	require.NoError(t, err)
 }
 
@@ -2359,7 +2359,7 @@ func TestAuthorizeSecurityGroupIngress_AssignsRuleIDs(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 
-	entry, err := svc.sgKV.Get(t.Context(), kvutil.AccountKey(testAccountID, sgID))
+	entry, err := svc.sgKV.Get(t.Context(), kvstore.AccountKey(testAccountID, sgID))
 	require.NoError(t, err)
 	var record SecurityGroupRecord
 	require.NoError(t, json.Unmarshal(entry.Value(), &record))

spinifex/domains/ec2/vpc/service_impl.go

@@ -26,7 +26,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/zoneid"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -173,7 +173,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("create JetStream client: %w", err)
 	}
 
-	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketVPCs, 10)
+	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketVPCs, err)
 	}
@@ -181,7 +181,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketVPCs, err)
 	}
 
-	subnetKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketSubnets, 10)
+	subnetKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketSubnets, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSubnets, err)
 	}
@@ -189,7 +189,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketSubnets, err)
 	}
 
-	vniKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketVNICounter, 10)
+	vniKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketVNICounter, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketVNICounter, err)
 	}
@@ -197,7 +197,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketVNICounter, err)
 	}
 
-	eniKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketENIs, 10)
+	eniKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketENIs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketENIs, err)
 	}
@@ -205,7 +205,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketENIs, err)
 	}
 
-	sgKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketSecurityGroups, 10)
+	sgKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketSecurityGroups, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSecurityGroups, err)
 	}
@@ -213,7 +213,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketSecurityGroups, err)
 	}
 
-	defaultsKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketDefaultVPCs, 10)
+	defaultsKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketDefaultVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketDefaultVPCs, err)
 	}
@@ -221,18 +221,18 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketDefaultVPCs, err)
 	}
 
-	rtbKV, err := kvutil.GetOrCreateBucket(ctx, js, "spinifex-vpc-route-tables", 10)
+	rtbKV, err := kvstore.GetOrCreateBucket(ctx, js, "spinifex-vpc-route-tables", 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket spinifex-vpc-route-tables: %w", err)
 	}
 
 	// Named literally, like the route table bucket above: the IGW package
 	// imports this one, so its constant cannot be referenced here.
-	igwKV, err := kvutil.GetOrCreateBucket(ctx, js, "spinifex-igw", 10)
+	igwKV, err := kvstore.GetOrCreateBucket(ctx, js, "spinifex-igw", 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket spinifex-igw: %w", err)
 	}
-	eigwKV, err := kvutil.GetOrCreateBucket(ctx, js, "spinifex-eigw", 10)
+	eigwKV, err := kvstore.GetOrCreateBucket(ctx, js, "spinifex-eigw", 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket spinifex-eigw: %w", err)
 	}
@@ -268,7 +268,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 // nextVNI allocates the next VNI using atomic increment on the NATS KV counter.
 func (s *VPCServiceImpl) nextVNI(ctx context.Context) (int64, error) {
 	var allocated int64
-	_, err := kvutil.Update(ctx, s.vniKV, vniCounterKey, kvutil.CASConfig{CreateIfAbsent: true, Attempts: vniCASAttempts},
+	_, err := kvstore.Update(ctx, s.vniKV, vniCounterKey, kvstore.CASConfig{CreateIfAbsent: true, Attempts: vniCASAttempts},
 		func(next *int64) (bool, error) {
 			if *next == 0 {
 				*next = vniStart
@@ -331,7 +331,7 @@ func (s *VPCServiceImpl) CreateVpc(ctx context.Context, input *ec2.CreateVpcInpu
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal VPC record: %w", err)
 	}
-	if _, err := s.vpcKV.Put(ctx, kvutil.AccountKey(accountID, vpcID), data); err != nil {
+	if _, err := s.vpcKV.Put(ctx, kvstore.AccountKey(accountID, vpcID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -366,7 +366,7 @@ func (s *VPCServiceImpl) CreateVpc(ctx context.Context, input *ec2.CreateVpcInpu
 // requireVPCExists returns InvalidVpcID.NotFound if the VPC doesn't exist for
 // this account.
 func (s *VPCServiceImpl) requireVPCExists(ctx context.Context, accountID, vpcId string) error {
-	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcId)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcId)); err != nil {
 		return vpcNotFoundError(vpcId)
 	}
 	return nil
@@ -379,7 +379,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 	}
 
 	vpcID := *input.VpcId
-	key := kvutil.AccountKey(accountID, vpcID)
+	key := kvstore.AccountKey(accountID, vpcID)
 
 	if _, err := s.vpcKV.Get(ctx, key); err != nil {
 		// AWS-faithful: an absent VPC is NotFound (the tofu/SDK provider
@@ -398,7 +398,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range subnetKeys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -435,7 +435,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range sgKeys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, k)
@@ -472,7 +472,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range rtbKeys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.rtbKV.Get(ctx, k)
@@ -533,7 +533,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	if rtbID != "" {
-		if err := s.rtbKV.Delete(ctx, kvutil.AccountKey(accountID, rtbID)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
+		if err := s.rtbKV.Delete(ctx, kvstore.AccountKey(accountID, rtbID)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
 			slog.ErrorContext(ctx, "DeleteVpc: main route table reap failed", "vpcId", vpcID, "routeTableId", rtbID, "err", err)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
@@ -574,7 +574,7 @@ func rejectAttachedGateway(ctx context.Context, kv jetstream.KeyValue, accountID
 	}
 
 	for _, k := range keys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := kv.Get(ctx, k)
@@ -653,7 +653,7 @@ func (s *VPCServiceImpl) DescribeVpcs(ctx context.Context, input *ec2.DescribeVp
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -723,7 +723,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
 	}
 
 	// Verify VPC exists and belongs to this account
-	vpcEntry, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID))
+	vpcEntry, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID))
 	if err != nil {
 		return nil, vpcNotFoundError(vpcID)
 	}
@@ -757,7 +757,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
 	}
 
 	for _, k := range subnetKeys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -822,7 +822,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal subnet record: %w", err)
 	}
-	if _, err := s.subnetKV.Put(ctx, kvutil.AccountKey(accountID, subnetID), data); err != nil {
+	if _, err := s.subnetKV.Put(ctx, kvstore.AccountKey(accountID, subnetID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -843,7 +843,7 @@ func (s *VPCServiceImpl) DeleteSubnet(ctx context.Context, input *ec2.DeleteSubn
 	}
 
 	subnetID := *input.SubnetId
-	key := kvutil.AccountKey(accountID, subnetID)
+	key := kvstore.AccountKey(accountID, subnetID)
 
 	// Read subnet record before deletion (needed for vpcd event)
 	subnetEntry, err := s.subnetKV.Get(ctx, key)
@@ -902,7 +902,7 @@ func (s *VPCServiceImpl) checkSubnetResidents(ctx context.Context, accountID, su
 	}
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, key)
@@ -950,7 +950,7 @@ func (s *VPCServiceImpl) clearRouteTableAssociationsForSubnet(ctx context.Contex
 	}
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.rtbKV.Get(ctx, key)
@@ -1035,7 +1035,7 @@ func (s *VPCServiceImpl) DescribeSubnets(ctx context.Context, input *ec2.Describ
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -1386,7 +1386,7 @@ func (s *VPCServiceImpl) ModifySubnetAttribute(ctx context.Context, input *ec2.M
 	}
 
 	subnetID := *input.SubnetId
-	key := kvutil.AccountKey(accountID, subnetID)
+	key := kvstore.AccountKey(accountID, subnetID)
 
 	entry, err := s.subnetKV.Get(ctx, key)
 	if err != nil {
@@ -1438,7 +1438,7 @@ func (s *VPCServiceImpl) ModifyVpcAttribute(ctx context.Context, input *ec2.Modi
 	}
 
 	vpcID := *input.VpcId
-	key := kvutil.AccountKey(accountID, vpcID)
+	key := kvstore.AccountKey(accountID, vpcID)
 
 	entry, err := s.vpcKV.Get(ctx, key)
 	if err != nil {
@@ -1486,7 +1486,7 @@ func (s *VPCServiceImpl) DescribeVpcAttribute(ctx context.Context, input *ec2.De
 	}
 
 	vpcID := *input.VpcId
-	key := kvutil.AccountKey(accountID, vpcID)
+	key := kvstore.AccountKey(accountID, vpcID)
 
 	entry, err := s.vpcKV.Get(ctx, key)
 	if err != nil {
@@ -1587,7 +1587,7 @@ func (s *VPCServiceImpl) writeMainRouteTable(ctx context.Context, accountID, vpc
 	if err != nil {
 		return false, fmt.Errorf("marshal main route table: %w", err)
 	}
-	if _, err := s.rtbKV.Create(ctx, kvutil.AccountKey(accountID, rtbID), data); err != nil {
+	if _, err := s.rtbKV.Create(ctx, kvstore.AccountKey(accountID, rtbID), data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
 			return false, nil
 		}
@@ -1611,7 +1611,7 @@ func (s *VPCServiceImpl) findMainRouteTableID(ctx context.Context, accountID, vp
 		return "", fmt.Errorf("list rtb keys: %w", err)
 	}
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.rtbKV.Get(ctx, key)
@@ -1652,7 +1652,7 @@ func (s *VPCServiceImpl) getDefaultSubnet(ctx context.Context, accountID string)
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -1680,7 +1680,7 @@ func (s *VPCServiceImpl) GetSubnet(accountID, subnetId string) (*SubnetRecord, e
 }
 
 func (s *VPCServiceImpl) getSubnet(ctx context.Context, accountID, subnetId string) (*SubnetRecord, error) {
-	key := kvutil.AccountKey(accountID, subnetId)
+	key := kvstore.AccountKey(accountID, subnetId)
 	entry, err := s.subnetKV.Get(ctx, key)
 	if err != nil {
 		return nil, fmt.Errorf("subnet %s not found: %w", subnetId, err)
@@ -1726,7 +1726,7 @@ func (s *VPCServiceImpl) SubnetAvailabilityZones(ctx context.Context) ([]string,
 	}
 	seen := map[string]struct{}{}
 	for _, k := range keys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		entry, err := s.subnetKV.Get(ctx, k)

spinifex/domains/ec2/vpc/service_impl_test.go

@@ -16,7 +16,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/zoneid"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
@@ -2689,7 +2689,7 @@ func TestCreateSubnet_ZoneStoreFailureWritesNothingAndPublishesNothing(t *testin
 		require.ErrorIs(t, err, jetstream.ErrNoKeysFound)
 	}
 	for _, k := range keys {
-		assert.Equal(t, kvutil.VersionKey, k, "no subnet record may exist")
+		assert.Equal(t, kvstore.VersionKey, k, "no subnet record may exist")
 	}
 	assert.Zero(t, events.Load(), "no topology event may be published")
 }

spinifex/domains/ecr/auth/ecrauth_test.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/golang-jwt/jwt/v5"
 	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -111,7 +111,7 @@ func TestLoadOrCreateSigningKey_FailsWhenNoKeyDecrypts(t *testing.T) {
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "master key does not match")
 
-	entries, err := kvutil.Keys(t.Context(), kv)
+	entries, err := kvstore.Keys(t.Context(), kv)
 	require.NoError(t, err)
 	assert.Len(t, entries, 1, "a failed load must not mint a key alongside the unreadable one")
 }

spinifex/domains/ecr/auth/signingkey.go

@@ -21,7 +21,7 @@ import (
 	"time"
 
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -111,7 +111,7 @@ func openSigningBucket(ctx context.Context, js jetstream.JetStream, masterKey []
 	if len(masterKey) == 0 {
 		return nil, errors.New("ecrauth: empty master key")
 	}
-	return kvutil.GetOrCreateBucket(ctx, js, SigningBucket, signingKeyHistory)
+	return kvstore.GetOrCreateBucket(ctx, js, SigningBucket, signingKeyHistory)
 }
 
 // reloadKeys reads every stored signing key, returning the verify set, each
@@ -125,7 +125,7 @@ func openSigningBucket(ctx context.Context, js jetstream.JetStream, masterKey []
 // load instead takes the gateway down on every node, permanently, over a key no
 // live request needs.
 func reloadKeys(ctx context.Context, kv jetstream.KeyValue, masterKey []byte) (active *SigningKey, verify map[string]*ecdsa.PublicKey, metas []keyMeta, undecodable int, err error) {
-	names, err := kvutil.Keys(ctx, kv)
+	names, err := kvstore.Keys(ctx, kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, nil, nil, 0, fmt.Errorf("list signing keys: %w", err)
 	}

spinifex/domains/ecr/meta.go

@@ -10,7 +10,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -179,7 +179,7 @@ func (s *KVMetaStore) bucket(ctx context.Context, accountID string) (jetstream.K
 		return kv, nil
 	}
 	name := KVAccountBucket(accountID)
-	kv, err := kvutil.GetOrCreateBucket(ctx, s.js, name, KVBucketAccountHistory)
+	kv, err := kvstore.GetOrCreateBucket(ctx, s.js, name, KVBucketAccountHistory)
 	if err != nil {
 		return nil, fmt.Errorf("ecr: open account bucket %s: %w", name, err)
 	}

spinifex/domains/ecs/store.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -158,7 +158,7 @@ func accountBuckets(ctx context.Context, nc *nats.Conn) ([]accountBucket, error)
 	if err != nil {
 		return nil, fmt.Errorf("jetstream: %w", err)
 	}
-	names, err := kvutil.BucketNames(ctx, js)
+	names, err := kvstore.BucketNames(ctx, js)
 	if err != nil {
 		return nil, err
 	}
@@ -176,7 +176,7 @@ func accountBuckets(ctx context.Context, nc *nats.Conn) ([]accountBucket, error)
 // return the existing handle.
 func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) {
 	bucket := AccountBucketName(accountID)
-	kv, err := kvutil.GetOrCreateBucket(ctx, js, bucket, KVBucketECSAccountHistory)
+	kv, err := kvstore.GetOrCreateBucket(ctx, js, bucket, KVBucketECSAccountHistory)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create ECS per-account KV bucket %s: %w", bucket, err)
 	}

spinifex/domains/eks/access/store.go

@@ -10,7 +10,7 @@ import (
 	"slices"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -128,7 +128,7 @@ func deleteRecord(ctx context.Context, kv jetstream.KeyValue, cluster, principal
 // casUpdate does a revision-checked read-modify-write. mutate returns true
 // when a field changed. Returns ErrNotFound if absent.
 func casUpdate(ctx context.Context, kv jetstream.KeyValue, cluster, principalARN string, mutate func(*Record) bool) (*Record, error) {
-	return kvutil.Update(ctx, kv, Key(cluster, principalARN), kvutil.CASConfig{NotFound: ErrNotFound}, func(r *Record) (bool, error) {
+	return kvstore.Update(ctx, kv, Key(cluster, principalARN), kvstore.CASConfig{NotFound: ErrNotFound}, func(r *Record) (bool, error) {
 		return mutate(r), nil
 	})
 }

spinifex/domains/eks/addon/store.go

@@ -8,7 +8,7 @@ import (
 	"slices"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -193,7 +193,7 @@ func deleteRecord(ctx context.Context, kv jetstream.KeyValue, cluster, addon str
 // casUpdate does a revision-checked read-modify-write. mutate returns true
 // when a field changed. Returns ErrNotFound if absent.
 func casUpdate(ctx context.Context, kv jetstream.KeyValue, cluster, addon string, mutate func(*Record) bool) (*Record, error) {
-	return kvutil.Update(ctx, kv, Key(cluster, addon), kvutil.CASConfig{NotFound: ErrNotFound}, func(r *Record) (bool, error) {
+	return kvstore.Update(ctx, kv, Key(cluster, addon), kvstore.CASConfig{NotFound: ErrNotFound}, func(r *Record) (bool, error) {
 		return mutate(r), nil
 	})
 }

spinifex/domains/eks/cluster/cluster_state.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -280,7 +280,7 @@ func casUpdateMeta(ctx context.Context, kv jetstream.KeyValue, name string, muta
 	if name == "" {
 		return errors.New("eks: casUpdateMeta empty name")
 	}
-	_, err := kvutil.Update(ctx, kv, ClusterMetaKey(name), kvutil.CASConfig{
+	_, err := kvstore.Update(ctx, kv, ClusterMetaKey(name), kvstore.CASConfig{
 		Attempts: maxClusterStateCASRetries,
 		NotFound: ErrClusterNotFound,
 		Exhausted: func(string, int) error {
@@ -435,7 +435,7 @@ func putIfGeneration(ctx context.Context, kv jetstream.KeyValue, meta *ClusterMe
 		if err == nil {
 			return nil
 		}
-		if !kvutil.IsRevisionConflict(err) {
+		if !kvstore.IsRevisionConflict(err) {
 			return fmt.Errorf("kv write %s: %w", key, err)
 		}
 	}

spinifex/domains/eks/cluster/cluster_teardown.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -292,7 +292,7 @@ func (s *EKSServiceImpl) removeClusterRecord(ctx context.Context, acctKV jetstre
 	}
 
 	errRefsAdded := errors.New("refs added")
-	_, err = kvutil.DeleteIf(ctx, acctKV, ClusterMetaKey(name), kvutil.CASConfig{
+	_, err = kvstore.DeleteIf(ctx, acctKV, ClusterMetaKey(name), kvstore.CASConfig{
 		Attempts: maxClusterStateCASRetries,
 		NotFound: ErrClusterNotFound,
 	}, func(m *ClusterMeta) error {

spinifex/domains/eks/cluster/cp_orphans.go

@@ -0,0 +1,158 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// ClusterRecordState is what a read of a cluster's record established.
+type ClusterRecordState int
+
+const (
+	ClusterRecordPresent ClusterRecordState = iota
+	ClusterRecordMissing
+	ClusterRecordUnreadable
+)
+
+// ClusterRecordVerdict is the outcome of looking a cluster record up without
+// creating anything. Reason names a missing or unreadable record; Err is set
+// only when the record could not be read.
+type ClusterRecordVerdict struct {
+	State  ClusterRecordState
+	Reason string
+	Err    error
+}
+
+// JudgeClusterRecord reads the cluster record read-only. A missing bucket or a
+// missing key is Missing, any other failure is Unreadable, and neither is ever
+// repaired. The billable reaper and the orphan report both use it, so they
+// judge a record the same way.
+func JudgeClusterRecord(ctx context.Context, js jetstream.JetStream, account, cluster string) ClusterRecordVerdict {
+	kv, err := OpenAccountBucket(ctx, js, account)
+	switch {
+	case errors.Is(err, jetstream.ErrBucketNotFound):
+		return ClusterRecordVerdict{State: ClusterRecordMissing, Reason: "account bucket missing"}
+	case err != nil:
+		return ClusterRecordVerdict{State: ClusterRecordUnreadable, Reason: "account bucket lookup failed", Err: err}
+	}
+	_, err = GetClusterMeta(ctx, kv, cluster)
+	switch {
+	case err == nil:
+		return ClusterRecordVerdict{State: ClusterRecordPresent}
+	case errors.Is(err, ErrClusterNotFound):
+		return ClusterRecordVerdict{State: ClusterRecordMissing, Reason: "cluster record missing"}
+	}
+	return ClusterRecordVerdict{State: ClusterRecordUnreadable, Reason: "cluster record read failed", Err: err}
+}
+
+// ENITagReader returns the tags on a control-plane ENI. found is false when
+// the ENI is gone.
+type ENITagReader func(ctx context.Context, eniID string) (tagSet map[string]string, found bool, err error)
+
+// ControlPlaneRow is one running EKS control-plane VM the report could not tie
+// to a live cluster record. Unreadable rows carry Err and are never omitted.
+type ControlPlaneRow struct {
+	InstanceID       string
+	Cluster          string
+	Account          string
+	Reason           string
+	Age              time.Duration
+	AvailabilityZone string
+	Unreadable       bool
+	Err              error
+}
+
+// FindControlPlaneOrphans classifies the instances in reservations. An instance
+// with no ENI, or whose ENI is gone or not tagged as an EKS control plane, is
+// skipped. A failed ENI read is an unreadable row, never a silent drop. A
+// control plane whose cluster record is present is omitted; one whose bucket or
+// record is missing is an orphan row; any other read failure is unreadable. It
+// only reads.
+func FindControlPlaneOrphans(ctx context.Context, js jetstream.JetStream, reservations []*ec2.Reservation, eni ENITagReader, now time.Time) []ControlPlaneRow {
+	var rows []ControlPlaneRow
+	for _, res := range reservations {
+		if res == nil {
+			continue
+		}
+		for _, inst := range res.Instances {
+			if inst == nil || !instanceRunning(inst) || len(inst.NetworkInterfaces) == 0 || inst.NetworkInterfaces[0] == nil {
+				continue
+			}
+			eniID := aws.StringValue(inst.NetworkInterfaces[0].NetworkInterfaceId)
+			if eniID == "" {
+				continue
+			}
+			row := ControlPlaneRow{InstanceID: aws.StringValue(inst.InstanceId)}
+			if inst.Placement != nil {
+				row.AvailabilityZone = aws.StringValue(inst.Placement.AvailabilityZone)
+			}
+			if inst.LaunchTime != nil {
+				row.Age = now.Sub(*inst.LaunchTime).Truncate(time.Second)
+			}
+
+			tagSet, found, err := eni(ctx, eniID)
+			if err != nil {
+				row.Unreadable, row.Err, row.Reason = true, err, "control-plane ENI read failed"
+				rows = append(rows, row)
+				continue
+			}
+			if !found || tagSet[tags.ManagedByKey] != tags.ManagedByEKS || tagSet[clusterEKSRoleTagKey] != clusterEKSRoleControlPlane {
+				continue
+			}
+			row.Cluster, row.Account = tagSet[clusterEKSClusterTagKey], tagSet[clusterEKSAccountTagKey]
+			if row.Cluster == "" || row.Account == "" {
+				continue
+			}
+
+			verdict := JudgeClusterRecord(ctx, js, row.Account, row.Cluster)
+			switch verdict.State {
+			case ClusterRecordPresent:
+				continue
+			case ClusterRecordUnreadable:
+				row.Unreadable, row.Err = true, verdict.Err
+			}
+			row.Reason = verdict.Reason
+			rows = append(rows, row)
+		}
+	}
+	sort.Slice(rows, func(i, j int) bool { return rows[i].InstanceID < rows[j].InstanceID })
+	return rows
+}
+
+// ReportControlPlaneOrphans writes the orphan report. list must return every
+// non-terminated system instance or an error: a failed or partial listing is
+// returned as an error and nothing is printed, so a short list is never taken
+// for a complete one. Unreadable rows are printed and counted separately.
+func ReportControlPlaneOrphans(ctx context.Context, w io.Writer, js jetstream.JetStream, list func(context.Context) (*ec2.DescribeInstancesOutput, error), eni ENITagReader, now time.Time) (orphans, unreadable int, err error) {
+	out, err := list(ctx)
+	if err != nil {
+		return 0, 0, fmt.Errorf("listing incomplete: %w", err)
+	}
+	if out == nil {
+		return 0, 0, errors.New("listing incomplete: no result")
+	}
+	rows := FindControlPlaneOrphans(ctx, js, out.Reservations, eni, now)
+	for _, r := range rows {
+		if r.Unreadable {
+			unreadable++
+			fmt.Fprintf(w, "UNREADABLE  %s  cluster=%s account=%s az=%s age=%s  %s: %v\n", r.InstanceID, r.Cluster, r.Account, r.AvailabilityZone, r.Age, r.Reason, r.Err)
+			continue
+		}
+		orphans++
+		fmt.Fprintf(w, "ORPHAN      %s  cluster=%s account=%s az=%s age=%s  %s\n", r.InstanceID, r.Cluster, r.Account, r.AvailabilityZone, r.Age, r.Reason)
+	}
+	return orphans, unreadable, nil
+}
+
+func instanceRunning(inst *ec2.Instance) bool {
+	return inst.State != nil && aws.StringValue(inst.State.Name) == ec2.InstanceStateNameRunning
+}

spinifex/domains/eks/cluster/cp_orphans_test.go

@@ -0,0 +1,198 @@
+package cluster
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ec2inventory "github.com/mulgadc/spinifex/spinifex/domains/ec2/inventory"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func orphanInstance(id, eni string) *ec2.Instance {
+	return &ec2.Instance{
+		InstanceId:        aws.String(id),
+		State:             &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)},
+		LaunchTime:        aws.Time(time.Unix(0, 0)),
+		Placement:         &ec2.Placement{AvailabilityZone: aws.String("ap-southeast-2a")},
+		NetworkInterfaces: []*ec2.InstanceNetworkInterface{{NetworkInterfaceId: aws.String(eni)}},
+	}
+}
+
+func cpTags(cluster, account string) map[string]string {
+	return map[string]string{
+		tags.ManagedByKey:       tags.ManagedByEKS,
+		clusterEKSClusterTagKey: cluster,
+		clusterEKSAccountTagKey: account,
+		clusterEKSRoleTagKey:    clusterEKSRoleControlPlane,
+	}
+}
+
+func workerTags(cluster, account string) map[string]string {
+	m := cpTags(cluster, account)
+	m[clusterEKSRoleTagKey] = "worker"
+	return m
+}
+
+func TestFindControlPlaneOrphansUnreadableRecordIsNotAnOrphan(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	eni := func(context.Context, string) (map[string]string, bool, error) {
+		return cpTags("gone", testAccountID), true, nil
+	}
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{orphanInstance("i-1", "eni-1")}}}
+
+	rows := FindControlPlaneOrphans(ctx, js, res, eni, time.Unix(60, 0))
+
+	require.Len(t, rows, 1)
+	assert.True(t, rows[0].Unreadable, "a read that failed is not a missing record")
+	assert.Error(t, rows[0].Err)
+}
+
+func TestFindControlPlaneOrphans(t *testing.T) {
+	const lost = "999999999999"
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("alive")))
+
+	eniTags := map[string]map[string]string{
+		"eni-orphan":   cpTags("gone", testAccountID),
+		"eni-live":     cpTags("alive", testAccountID),
+		"eni-nobucket": cpTags("lost", lost),
+		"eni-worker":   workerTags("gone", testAccountID),
+	}
+	eni := func(_ context.Context, id string) (map[string]string, bool, error) {
+		if id == "eni-broken" {
+			return nil, false, errors.New("vpc service down")
+		}
+		m, ok := eniTags[id]
+		return m, ok, nil
+	}
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
+		orphanInstance("i-orphan", "eni-orphan"),
+		orphanInstance("i-live", "eni-live"),
+		orphanInstance("i-nobucket", "eni-nobucket"),
+		orphanInstance("i-worker", "eni-worker"),
+		orphanInstance("i-ghost", "eni-ghost"),
+		orphanInstance("i-broken", "eni-broken"),
+		{InstanceId: aws.String("i-noeni")},
+	}}}
+
+	rows := FindControlPlaneOrphans(t.Context(), js, res, eni, time.Unix(60, 0))
+
+	got := map[string]ControlPlaneRow{}
+	for _, r := range rows {
+		got[r.InstanceID] = r
+	}
+	assert.Len(t, got, 3, "only orphan, nobucket and the unreadable ENI are reported")
+	assert.Equal(t, "cluster record missing", got["i-orphan"].Reason)
+	assert.False(t, got["i-orphan"].Unreadable)
+	assert.Equal(t, "account bucket missing", got["i-nobucket"].Reason)
+	assert.True(t, got["i-broken"].Unreadable, "a failed ENI read is never dropped")
+	assert.Error(t, got["i-broken"].Err)
+	assert.NotContains(t, got, "i-live", "a present record is omitted")
+	assert.NotContains(t, got, "i-worker")
+	assert.NotContains(t, got, "i-ghost")
+	assert.NotContains(t, got, "i-noeni")
+}
+
+func TestReportControlPlaneOrphansIncompleteListingPrintsNothing(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	var buf bytes.Buffer
+	list := func(context.Context) (*ec2.DescribeInstancesOutput, error) {
+		return nil, errors.New("ServiceUnavailable")
+	}
+	orphans, unreadable, err := ReportControlPlaneOrphans(t.Context(), &buf, js, list, nil, time.Now())
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "listing incomplete")
+	assert.Zero(t, orphans+unreadable)
+	assert.Empty(t, buf.String(), "no partial list on a failed listing")
+}
+
+func TestReportControlPlaneOrphansPrintsRows(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	var buf bytes.Buffer
+	list := func(context.Context) (*ec2.DescribeInstancesOutput, error) {
+		return &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{
+			orphanInstance("i-orphan", "eni-orphan"), orphanInstance("i-broken", "eni-broken"),
+		}}}}, nil
+	}
+	eni := func(_ context.Context, id string) (map[string]string, bool, error) {
+		if id == "eni-broken" {
+			return nil, false, errors.New("boom")
+		}
+		return cpTags("gone", testAccountID), true, nil
+	}
+	orphans, unreadable, err := ReportControlPlaneOrphans(t.Context(), &buf, js, list, eni, time.Unix(60, 0))
+	require.NoError(t, err)
+	assert.Equal(t, 1, orphans)
+	assert.Equal(t, 1, unreadable)
+	assert.Contains(t, buf.String(), "ORPHAN      i-orphan")
+	assert.Contains(t, buf.String(), "UNREADABLE  i-broken")
+	assert.Equal(t, 2, strings.Count(buf.String(), "az=ap-southeast-2a"), "both row kinds print the zone")
+}
+
+func TestFindControlPlaneOrphansCorruptENIRecordIsUnreadable(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	enis, err := kvstore.GetOrCreateBucket(t.Context(), js, ec2vpc.KVBucketENIs, 10)
+	require.NoError(t, err)
+	good, err := json.Marshal(ec2vpc.ENIRecord{NetworkInterfaceId: "eni-good", Tags: cpTags("gone", testAccountID)})
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvstore.AccountKey("000000000000", "eni-good"), good)
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvstore.AccountKey("000000000000", "eni-corrupt"), []byte("{not json"))
+	require.NoError(t, err)
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
+		orphanInstance("i-good", "eni-good"), orphanInstance("i-corrupt", "eni-corrupt"), orphanInstance("i-absent", "eni-absent"),
+	}}}
+
+	rows := FindControlPlaneOrphans(t.Context(), js, res, ec2inventory.NewENITagReader(js, "000000000000"), time.Unix(60, 0))
+
+	require.Len(t, rows, 2, "an absent ENI is skipped, the other two are reported")
+	assert.Equal(t, "i-corrupt", rows[0].InstanceID)
+	assert.True(t, rows[0].Unreadable, "a corrupt ENI record is unreadable, never gone")
+	assert.Equal(t, "i-good", rows[1].InstanceID)
+	assert.False(t, rows[1].Unreadable)
+}
+
+func TestFindControlPlaneOrphansOnlyReportsRunningInstances(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	eni := func(context.Context, string) (map[string]string, bool, error) {
+		return cpTags("gone", testAccountID), true, nil
+	}
+	mk := func(id, state string) *ec2.Instance {
+		i := orphanInstance(id, "eni-"+id)
+		i.State = &ec2.InstanceState{Name: aws.String(state)}
+		return i
+	}
+	noState := orphanInstance("i-nostate", "eni-nostate")
+	noState.State = nil
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
+		mk("i-running", ec2.InstanceStateNameRunning),
+		mk("i-stopped", ec2.InstanceStateNameStopped),
+		mk("i-pending", ec2.InstanceStateNamePending),
+		noState,
+	}}}
+
+	rows := FindControlPlaneOrphans(t.Context(), js, res, eni, time.Unix(60, 0))
+
+	require.Len(t, rows, 1)
+	assert.Equal(t, "i-running", rows[0].InstanceID)
+}

spinifex/domains/eks/cluster/eks_billable_reaper.go

@@ -2,7 +2,6 @@ package cluster
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"log/slog"
 
@@ -13,13 +12,10 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 )
 
-// EKSBillableReaper is the meta-independent billable cleanup from historic PROP-LIFECYCLE-006 §5 (behaviour per ADR-0003:S2), built on
-// the resource lifecycle GC backstop. A normal DeleteCluster can leave a
-// running control-plane VM behind (terminate completes async; a daemon restart
-// re-launches it from saved running-state) after the cluster meta is already
-// swept — a billable orphan with no owner to drive a retry. This reaper finds a
-// running EKS control-plane VM whose cluster meta is DEFINITIVELY GONE and
-// terminates it. It is node-local: each node reaps the orphans it runs.
+// EKSBillableReaper raises the data-safety alarm for a running EKS control-plane
+// VM whose cluster record cannot be found. It never terminates anything: a
+// missing record cannot tell a completed DeleteCluster from a lost bucket or an
+// out-of-band delete, so removal is an operator decision. It is node-local.
 type EKSBillableReaper struct {
 	svc *EKSServiceImpl
 	// running returns this node's running VMs. The reaper filters to the EKS
@@ -38,10 +34,11 @@ func (s *EKSServiceImpl) NewBillableReaper(running func() ([]*vm.VM, error)) *EK
 func (r *EKSBillableReaper) Class() string         { return "eks-billable" }
 func (r *EKSBillableReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
 
-// Sweep terminates every running EKS control-plane VM whose cluster meta is
-// absent. A VM whose cluster still exists (CREATING or DELETING) is left to the
-// cluster's own teardown/retry; an ENI that is gone or untagged is skipped, so
-// the reaper never reaps on uncertainty.
+// Sweep alarms on every running EKS control-plane VM whose cluster record, or
+// whole account bucket, is missing, and removes nothing. A VM whose cluster
+// exists is left to the cluster's own teardown; an ENI that is gone or untagged
+// is skipped, and a read that fails is logged and skipped. It returns 0 because
+// it never removes a VM.
 func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 	vms, err := r.running()
 	if err != nil {
@@ -53,11 +50,10 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 		return 0, fmt.Errorf("jetstream: %w", err)
 	}
 
-	reaped := 0
 	for _, v := range vms {
 		select {
 		case <-ctx.Done():
-			return reaped, ctx.Err()
+			return 0, ctx.Err()
 		default:
 		}
 		if v == nil || v.ManagedBy != tags.ManagedByEKS || v.ENIId == "" {
@@ -66,47 +62,23 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 
 		clusterName, clusterAccount, ok := r.clusterRefFromENI(ctx, v)
 		if !ok {
-			continue // ENI gone or untagged: cannot confirm orphan-hood, never reap
+			continue // ENI gone or untagged: cannot tie the VM to a cluster
 		}
 
-		acctKV, err := GetOrCreateAccountBucket(ctx, js, clusterAccount)
-		if err != nil {
-			slog.WarnContext(ctx, "eks-billable: account bucket lookup failed", "account", clusterAccount, "err", err)
-			continue
-		}
-		if _, err := GetClusterMeta(ctx, acctKV, clusterName); !errors.Is(err, ErrClusterNotFound) {
-			// Meta present (live/CREATING/DELETING) or unreadable: not a definitive
-			// orphan — leave it to the cluster's own teardown/retry.
+		verdict := JudgeClusterRecord(ctx, js, clusterAccount, clusterName)
+		switch verdict.State {
+		case ClusterRecordPresent:
+			continue // cluster exists (live, CREATING or DELETING)
+		case ClusterRecordUnreadable:
+			slog.WarnContext(ctx, "eks-billable: "+verdict.Reason, "cluster", clusterName, "account", clusterAccount, "err", verdict.Err)
 			continue
 		}
+		reason := verdict.Reason
 
-		// ALARM + reap: the cluster is gone but its control-plane VM still runs.
-		slog.WarnContext(ctx, "DATA-SAFETY ALARM: orphaned EKS control-plane VM reaped (cluster meta gone)",
-			"instanceId", v.ID, "cluster", clusterName, "account", clusterAccount, "node", v.LastNode)
-		if err := TerminateK3sServerVM(ctx, r.svc.deps.VPCK3s, r.svc.deps.Instance, v.AccountID, v.ID, v.ENIId); err != nil {
-			slog.WarnContext(ctx, "eks-billable: failed to terminate orphan CP VM", "instanceId", v.ID, "err", err)
-			continue
-		}
-		reaped++
-		r.reclaimOrphanInfra(ctx, v.AccountID, clusterName)
-	}
-	return reaped, nil
-}
-
-// reclaimOrphanInfra tears down the billable infra a reaped orphan's swept meta
-// no longer anchors: the NLB front-end NAT and the managed CP VPC's NAT-GW EIP.
-// Both are keyed by cluster name and idempotent, so a clean prior delete is a
-// no-op. Best-effort — the costliest resource (the VM) is already gone, and
-// DeleteClusterCPVPC releases the NAT-GW EIP before the VPC delete, so the
-// billable address is reclaimed even if the now-empty VPC delete trails. account
-// is the infra account (system account for the managed CP VPC topology).
-func (r *EKSBillableReaper) reclaimOrphanInfra(ctx context.Context, account, clusterName string) {
-	if err := DeleteClusterNLB(ctx, r.svc.deps.NLB, account, clusterName); err != nil {
-		slog.WarnContext(ctx, "eks-billable: reclaim orphan NLB failed", "cluster", clusterName, "err", err)
-	}
-	if err := DeleteClusterCPVPC(ctx, r.svc.cpVPCDeps(), account, clusterName, nil); err != nil {
-		slog.WarnContext(ctx, "eks-billable: reclaim orphan CP VPC (NAT-GW EIP) failed", "cluster", clusterName, "err", err)
+		slog.WarnContext(ctx, "DATA-SAFETY ALARM: EKS control-plane VM has no cluster record; retained for an operator",
+			"reason", reason, "instanceId", v.ID, "cluster", clusterName, "account", clusterAccount, "node", v.LastNode)
 	}
+	return 0, nil
 }
 
 // clusterRefFromENI reads the cluster name + customer account from the VM's

spinifex/domains/eks/cluster/eks_billable_reaper_test.go

@@ -1,14 +1,19 @@
 package cluster
 
 import (
+	"bytes"
 	"context"
+	"log/slog"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/elbv2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -33,53 +38,119 @@ func cpVM(id, eniID string) *vm.VM {
 	return &vm.VM{ID: id, ManagedBy: tags.ManagedByEKS, ENIId: eniID, AccountID: reaperSysAccount, LastNode: "node-1"}
 }
 
-// TestRLC5_EKSBillableReaperTerminatesOrphanCPVM exercises the historic PROP-LIFECYCLE-006 §5 (behaviour per ADR-0003:S2)
-// meta-independent billable cleanup: a running EKS control-plane VM whose
-// cluster meta is DEFINITIVELY GONE is a billable orphan and must be terminated
-// by the GC backstop — the real fix for the orphan CP VM surviving a
-// daemon restart after DeleteCluster swept the meta.
-func TestRLC5_EKSBillableReaperTerminatesOrphanCPVM(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	f.vpc.describeByENI = map[string]*ec2.NetworkInterface{
-		"eni-orphan": cpENI("eni-orphan", "gone-cluster", testAccountID),
+// ADR-0003:S7. A running control-plane VM whose cluster record cannot be found
+// is retained and alarmed, never terminated and never has its NLB or VPC torn
+// down: a missing record cannot tell a completed DeleteCluster from a lost
+// bucket or an out-of-band delete.
+func TestS7_EKSBillableReaperNeverTerminatesAControlPlane(t *testing.T) {
+	const lostAccount = "999999999999"
+
+	cases := []struct {
+		name      string
+		account   string
+		prepare   func(t *testing.T, f *eksServiceFixture)
+		wantAlarm string
+		wantLog   string
+	}{
+		{
+			name:      "account bucket missing",
+			account:   lostAccount,
+			prepare:   func(*testing.T, *eksServiceFixture) {},
+			wantAlarm: "account bucket missing",
+		},
+		{
+			name:      "bucket has no meta key",
+			account:   testAccountID,
+			prepare:   func(*testing.T, *eksServiceFixture) {},
+			wantAlarm: "cluster record missing",
+		},
+		{
+			name:    "meta removed by the teardown revision-checked delete",
+			account: testAccountID,
+			prepare: func(t *testing.T, f *eksServiceFixture) {
+				require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("orphan")))
+				_, err := kvstore.DeleteIf(t.Context(), f.kv, ClusterMetaKey("orphan"), kvstore.CASConfig{
+					Attempts: maxClusterStateCASRetries,
+					NotFound: ErrClusterNotFound,
+				}, func(*ClusterMeta) error { return nil })
+				require.NoError(t, err)
+			},
+			wantAlarm: "cluster record missing",
+		},
+		{
+			name:    "meta removed out of band",
+			account: testAccountID,
+			prepare: func(t *testing.T, f *eksServiceFixture) {
+				require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("orphan")))
+				require.NoError(t, f.kv.Delete(t.Context(), ClusterMetaKey("orphan")))
+			},
+			wantAlarm: "cluster record missing",
+		},
+		{
+			name:    "meta unreadable",
+			account: testAccountID,
+			prepare: func(t *testing.T, f *eksServiceFixture) {
+				_, err := f.kv.Put(t.Context(), ClusterMetaKey("orphan"), []byte("{not json"))
+				require.NoError(t, err)
+			},
+			wantLog: "cluster record read failed",
+		},
 	}
-	orphan := cpVM("i-orphan", "eni-orphan")
 
-	// Seed the orphan cluster's name/tag-driven billable infra the swept meta no
-	// longer anchors: an NLB front-end and a tagged NAT gateway holding a billable
-	// EIP. The reaper's post-terminate reclaim must sweep both.
-	nlbName := ClusterNLBName("gone-cluster")
-	f.nlb.lbByName[nlbName] = &elbv2.LoadBalancer{
-		LoadBalancerArn:  aws.String("arn:lb-orphan"),
-		LoadBalancerName: aws.String(nlbName),
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			// slog.SetDefault is process-wide, so this test is not parallel.
+			var buf bytes.Buffer
+			restore := slog.Default()
+			slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
+			t.Cleanup(func() { slog.SetDefault(restore) })
+
+			f := newEKSServiceFixture(t)
+			f.vpc.describeByENI = map[string]*ec2.NetworkInterface{
+				"eni-cp": cpENI("eni-cp", "orphan", tc.account),
+			}
+			nlbName := ClusterNLBName("orphan")
+			f.nlb.lbByName[nlbName] = &elbv2.LoadBalancer{
+				LoadBalancerArn:  aws.String("arn:lb-orphan"),
+				LoadBalancerName: aws.String(nlbName),
+			}
+			f.ngw.gws = []*fakeCPNatGateway{{
+				id:    "nat-orphan",
+				tags:  map[string]string{clusterEKSClusterTagKey: "orphan", clusterEKSRoleTagKey: cpVPCRoles.NatGW},
+				state: "available",
+				addrs: []*ec2.NatGatewayAddress{{AllocationId: aws.String("eipalloc-orphan")}},
+			}}
+			tc.prepare(t, f)
+
+			reaper := f.svc.NewBillableReaper(func() ([]*vm.VM, error) { return []*vm.VM{cpVM("i-cp", "eni-cp")}, nil })
+			for range 2 {
+				removed, err := reaper.Sweep(context.Background())
+				require.NoError(t, err)
+				assert.Zero(t, removed, "ADR-0003:S7: the reaper removes nothing")
+			}
+
+			assert.Empty(t, f.inst.terminateCalls, "ADR-0003:S7: a reaper's judgement is never the decision to terminate")
+			assert.Empty(t, f.vpc.deleteCalls, "the control-plane ENI must stay")
+			assert.Contains(t, f.nlb.lbByName, nlbName, "the NLB must stay")
+			assert.Empty(t, f.eip.releaseCalls, "no EIP may be released")
+
+			js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+			if tc.account == lostAccount {
+				_, err := OpenAccountBucket(t.Context(), js, lostAccount)
+				assert.ErrorIs(t, err, jetstream.ErrBucketNotFound, "a sweep must not create the bucket it found missing")
+			}
+
+			out := buf.String()
+			if tc.wantAlarm != "" {
+				assert.Contains(t, out, "DATA-SAFETY ALARM")
+				assert.Contains(t, out, tc.wantAlarm)
+				assert.Contains(t, out, "i-cp")
+			} else {
+				assert.NotContains(t, out, "DATA-SAFETY ALARM", "an unreadable record is not a missing one")
+				assert.Contains(t, out, tc.wantLog)
+			}
+		})
 	}
-	f.ngw.gws = []*fakeCPNatGateway{{
-		id:    "nat-orphan",
-		tags:  map[string]string{clusterEKSClusterTagKey: "gone-cluster", clusterEKSRoleTagKey: cpVPCRoles.NatGW},
-		state: "available",
-		addrs: []*ec2.NatGatewayAddress{{AllocationId: aws.String("eipalloc-orphan")}},
-	}}
-
-	reaper := f.svc.NewBillableReaper(func() ([]*vm.VM, error) { return []*vm.VM{orphan}, nil })
-
-	reaped, err := reaper.Sweep(context.Background())
-	require.NoError(t, err)
-	assert.Equal(t, 1, reaped, "ADR-0003:S2 behaviour; historic PROP-LIFECYCLE-006 §5 mechanism: a CP VM whose cluster meta is gone must be reaped")
-	assert.Contains(t, f.inst.terminateCalls, "i-orphan", "the orphan CP VM must be terminated")
-	require.Len(t, f.vpc.deleteCalls, 1, "the orphan CP ENI must be deleted")
-	assert.Equal(t, "eni-orphan", aws.StringValue(f.vpc.deleteCalls[0].NetworkInterfaceId))
-
-	// The orphan's billable NLB + NAT-GW EIP must also be reclaimed.
-	assert.NotContains(t, f.nlb.lbByName, nlbName, "the orphan NLB front-end must be reclaimed")
-	require.Len(t, f.eip.releaseCalls, 1, "the orphan NAT-GW EIP must be released")
-	assert.Equal(t, "eipalloc-orphan", aws.StringValue(f.eip.releaseCalls[0].AllocationId))
-
-	// Idempotent: a repeat sweep over the same (still meta-absent) VM is safe —
-	// the terminate + reclaim paths tolerate already-gone infra (no second release).
-	_, err = reaper.Sweep(context.Background())
-	require.NoError(t, err)
-	assert.Len(t, f.eip.releaseCalls, 1, "reclaim must not double-release an already-freed EIP")
 }
 
 // TestRLC5_EKSBillableReaperSpareLiveAndUncertain enforces the reaper's no-false-

spinifex/domains/eks/cluster/service_impl.go

@@ -33,7 +33,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -2138,7 +2138,7 @@ func (s *EKSServiceImpl) casUpdateNodegroupTags(ctx context.Context, accountID,
 	if err != nil {
 		return err
 	}
-	_, err = kvutil.Update(ctx, acctKV, NodegroupKey(cluster, ng), kvutil.CASConfig{
+	_, err = kvstore.Update(ctx, acctKV, NodegroupKey(cluster, ng), kvstore.CASConfig{
 		Attempts:  ngCASMaxRetries,
 		NotFound:  errors.New(awserrors.ErrorEKSResourceNotFound),
 		Exhausted: func(string, int) error { return errors.New(awserrors.ErrorServerInternal) },

spinifex/domains/eks/cluster/store.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -157,7 +156,7 @@ func accountBucketNames(ctx context.Context, nc *nats.Conn) ([]string, error) {
 	if err != nil {
 		return nil, fmt.Errorf("jetstream: %w", err)
 	}
-	all, err := kvutil.BucketNames(ctx, js)
+	all, err := kvstore.BucketNames(ctx, js)
 	if err != nil {
 		return nil, err
 	}
@@ -185,7 +184,8 @@ func AccountWatchBuckets(ctx context.Context, nc *nats.Conn) ([]*kvstore.Bucket,
 	}
 	buckets := make([]*kvstore.Bucket, 0, len(names))
 	for _, name := range names {
-		buckets = append(buckets, kvstore.NewBucket(js, kvstore.Config{Name: name, History: 1}))
+		// Attach only: a watch on a deleted bucket must fail and retry, not recreate it.
+		buckets = append(buckets, kvstore.NewBucket(js, kvstore.Config{Name: name, History: 1, AttachOnly: true}))
 	}
 	return buckets, nil
 }
@@ -196,7 +196,7 @@ func AccountWatchBuckets(ctx context.Context, nc *nats.Conn) ([]*kvstore.Bucket,
 // existing handle.
 func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) {
 	bucket := AccountBucketName(accountID)
-	kv, err := kvutil.GetOrCreateBucket(ctx, js, bucket, KVBucketEKSAccountHistory)
+	kv, err := kvstore.GetOrCreateBucket(ctx, js, bucket, KVBucketEKSAccountHistory)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create EKS per-account KV bucket %s: %w", bucket, err)
 	}
@@ -206,6 +206,13 @@ func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accou
 	return kv, nil
 }
 
+// OpenAccountBucket returns the per-account KV bucket for accountID without
+// creating it. A bucket that does not exist is jetstream.ErrBucketNotFound, so a
+// read-only caller never turns a lost bucket into an empty one.
+func OpenAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) {
+	return js.KeyValue(ctx, AccountBucketName(accountID))
+}
+
 // InitLeaderBucket creates (or attaches to) the shared spinifex-eks-leader
 // bucket used for per-cluster reconciler leader-lease CAS locks.
 func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {

spinifex/domains/eks/cluster/watch_source_test.go

@@ -6,7 +6,8 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -23,7 +24,7 @@ func TestAccountWatchBuckets_ReturnsOnePerAccountBucket(t *testing.T) {
 	require.NoError(t, err)
 	_, err = ekscluster.GetOrCreateAccountBucket(t.Context(), js, "222222222222")
 	require.NoError(t, err)
-	_, err = kvutil.GetOrCreateBucket(t.Context(), js, "something-else", 1)
+	_, err = kvstore.GetOrCreateBucket(t.Context(), js, "something-else", 1)
 	require.NoError(t, err)
 
 	buckets, err := ekscluster.AccountWatchBuckets(t.Context(), nc)
@@ -79,3 +80,26 @@ func TestAccountWatchBuckets_EnumerationErrorPropagates(t *testing.T) {
 	_, err := ekscluster.AccountWatchBuckets(ctx, nc)
 	require.Error(t, err)
 }
+
+// TestAccountWatchBuckets_WatchOnDeletedBucketDoesNotRecreateIt pins attach-only
+// handles: a DNS watch that outlives its bucket must fail and retry, because
+// recreating it hides a deleted account bucket behind an empty one.
+func TestAccountWatchBuckets_WatchOnDeletedBucketDoesNotRecreateIt(t *testing.T) {
+	t.Parallel()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	js := testutil.NewJetStream(t, nc)
+	_, err := ekscluster.GetOrCreateAccountBucket(t.Context(), js, "111111111111")
+	require.NoError(t, err)
+
+	buckets, err := ekscluster.AccountWatchBuckets(t.Context(), nc)
+	require.NoError(t, err)
+	require.Len(t, buckets, 1)
+
+	require.NoError(t, js.DeleteKeyValue(t.Context(), ekscluster.AccountBucketName("111111111111")))
+
+	_, err = buckets[0].Watch(t.Context(), ">")
+	require.Error(t, err, "a watch on a deleted bucket must fail")
+
+	_, err = ekscluster.OpenAccountBucket(t.Context(), js, "111111111111")
+	assert.ErrorIs(t, err, jetstream.ErrBucketNotFound, "the watch must not have recreated the bucket")
+}

spinifex/domains/elbv2/agent_heartbeat_wire_test.go

@@ -0,0 +1,73 @@
+package elbv2
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/query"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLBAgentHeartbeatInput_OldAgentDecodesUnchanged(t *testing.T) {
+	t.Parallel()
+	// Exactly the parameters an agent without the data-path probe sends.
+	params := map[string]string{
+		"Action":                   "LBAgentHeartbeat",
+		"Version":                  "2015-12-01",
+		"LBID":                     "lb-abc123",
+		"Servers.member.1.Backend": "bk_tg1",
+		"Servers.member.1.Server":  "srv_i-web1",
+		"Servers.member.1.Status":  "UP",
+		"Servers.member.2.Backend": "bk_tg1",
+		"Servers.member.2.Server":  "srv_i-web2",
+		"Servers.member.2.Status":  "DOWN",
+	}
+
+	var in LBAgentHeartbeatInput
+	require.NoError(t, query.QueryParamsToStruct(params, &in))
+
+	require.NotNil(t, in.LBID)
+	assert.Equal(t, "lb-abc123", *in.LBID)
+	require.Len(t, in.Servers, 2)
+	assert.Equal(t, "DOWN", *in.Servers[1].Status)
+	assert.Nil(t, in.InstanceID, "an old agent reports no instance ID")
+	assert.Empty(t, in.NICs, "an old agent reports no NICs")
+	assert.Len(t, in.toHealthReport().Servers, 2)
+}
+
+func TestLBAgentHeartbeatInput_DecodesDataPathEvidence(t *testing.T) {
+	t.Parallel()
+	params := map[string]string{
+		"LBID":                              "lb-abc123",
+		"InstanceID":                        "i-0123456789abcdef0",
+		"NICs.member.1.MAC":                 "02:00:00:aa:00:01",
+		"NICs.member.1.ProbeOK":             "false",
+		"NICs.member.1.ConsecutiveFailures": "4",
+		"NICs.member.1.RxPackets":           "100",
+		"NICs.member.1.TxPackets":           "200",
+		"NICs.member.1.RxDropped":           "3",
+		"NICs.member.1.TxDropped":           "9",
+		"NICs.member.2.MAC":                 "02:00:00:aa:00:03",
+		"NICs.member.2.ProbeOK":             "true",
+		"NICs.member.2.ConsecutiveFailures": "0",
+	}
+
+	var in LBAgentHeartbeatInput
+	require.NoError(t, query.QueryParamsToStruct(params, &in))
+
+	require.NotNil(t, in.InstanceID)
+	assert.Equal(t, "i-0123456789abcdef0", *in.InstanceID)
+	require.Len(t, in.NICs, 2)
+
+	n := in.NICs[0]
+	assert.Equal(t, "02:00:00:aa:00:01", *n.MAC)
+	assert.False(t, *n.ProbeOK)
+	assert.EqualValues(t, 4, *n.ConsecutiveFailures)
+	assert.EqualValues(t, 100, *n.RxPackets)
+	assert.EqualValues(t, 200, *n.TxPackets)
+	assert.EqualValues(t, 3, *n.RxDropped)
+	assert.EqualValues(t, 9, *n.TxDropped)
+
+	assert.True(t, *in.NICs[1].ProbeOK)
+	assert.Nil(t, in.NICs[1].RxPackets, "an omitted counter stays unset")
+}

spinifex/domains/elbv2/agent_types.go

@@ -7,9 +7,25 @@ import (
 
 // LBAgentHeartbeatInput is sent by the LB agent on each heartbeat tick,
 // including its health report so the daemon can update target health.
+//
+// InstanceID and NICs are optional: an older agent sends neither.
 type LBAgentHeartbeatInput struct {
-	LBID    *string                `locationName:"LBID" type:"string"`
-	Servers []*LBAgentServerStatus `locationName:"Servers" type:"list"`
+	LBID       *string                `locationName:"LBID" type:"string"`
+	Servers    []*LBAgentServerStatus `locationName:"Servers" type:"list"`
+	InstanceID *string                `locationName:"InstanceID" type:"string"`
+	NICs       []*LBAgentNICStatus    `locationName:"NICs" type:"list"`
+}
+
+// LBAgentNICStatus is the gateway probe result and interface counters of one
+// data NIC, as reported by the LB agent.
+type LBAgentNICStatus struct {
+	MAC                 *string `locationName:"MAC" type:"string"`
+	ProbeOK             *bool   `locationName:"ProbeOK" type:"boolean"`
+	ConsecutiveFailures *int64  `locationName:"ConsecutiveFailures" type:"long"`
+	RxPackets           *int64  `locationName:"RxPackets" type:"long"`
+	TxPackets           *int64  `locationName:"TxPackets" type:"long"`
+	RxDropped           *int64  `locationName:"RxDropped" type:"long"`
+	TxDropped           *int64  `locationName:"TxDropped" type:"long"`
 }
 
 // LBAgentServerStatus represents a single backend server's health status as reported by the LB agent.

spinifex/domains/elbv2/launcher_test.go

@@ -14,6 +14,9 @@ type mockSystemInstanceLauncher struct {
 	launchErr      error
 	terminateErr   error
 	terminateDone  chan struct{} // closed after TerminateSystemInstance completes
+	onTerminate    func(instanceID string)
+	rebootCalls    []string
+	rebootErr      error
 }
 
 func (m *mockSystemInstanceLauncher) LaunchSystemInstance(input *SystemInstanceInput) (*SystemInstanceOutput, error) {
@@ -25,12 +28,22 @@ func (m *mockSystemInstanceLauncher) TerminateSystemInstance(instanceID string)
 	m.mu.Lock()
 	m.terminateCalls = append(m.terminateCalls, instanceID)
 	m.mu.Unlock()
+	if m.onTerminate != nil {
+		m.onTerminate(instanceID)
+	}
 	if m.terminateDone != nil {
 		close(m.terminateDone)
 	}
 	return m.terminateErr
 }
 
+func (m *mockSystemInstanceLauncher) RebootSystemInstance(instanceID string) error {
+	m.mu.Lock()
+	m.rebootCalls = append(m.rebootCalls, instanceID)
+	m.mu.Unlock()
+	return m.rebootErr
+}
+
 // waitTerminate blocks until TerminateSystemInstance has been called.
 func (m *mockSystemInstanceLauncher) waitTerminate() {
 	if m.terminateDone != nil {

spinifex/domains/elbv2/lb_converge_test.go

@@ -0,0 +1,192 @@
+package elbv2
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLBRepair_AnotherReservationWinsTheFirstCAS(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	lb := e.seed("lbr20", true)
+	cands := e.candidates()
+	require.Len(t, cands, 1)
+	require.True(t, e.rec.eligible(context.Background(), cands[0], e.clock))
+
+	_, err := e.svc.store.UpdateLBHealth(context.Background(), lb.LoadBalancerID, func(rec *LBDataPathRecord) (bool, error) {
+		rec.Repairs = append(rec.Repairs, LBRepair{At: e.clock, InstanceID: lb.InstanceID, Outcome: RepairReserved})
+		return true, nil
+	})
+	require.NoError(t, err)
+
+	assert.False(t, e.rec.repair(context.Background(), cands[0]))
+	assert.Empty(t, e.reboots())
+	assert.Len(t, e.health(lb.LoadBalancerID).Repairs, 1, "the losing reservation must not add an entry")
+}
+
+func TestLBRepair_LeaseLostAfterReservationKeepsItCountedAndDoesNotDispatch(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	e.seed("lbr21", true)
+	cands := e.candidates()
+	require.Len(t, cands, 1)
+
+	calls := 0
+	e.rec.isLeader = func() bool {
+		calls++
+		return calls == 1
+	}
+	assert.True(t, e.rec.repair(context.Background(), cands[0]), "a written reservation is reported as spent")
+	assert.Empty(t, e.reboots())
+
+	repairs := e.health("lbr21").Repairs
+	require.Len(t, repairs, 1)
+	assert.Equal(t, RepairReserved, repairs[0].Outcome, "the entry stays counted with no outcome")
+}
+
+func TestLBRepair_NonLeaderNeverReserves(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	e.seed("lbr22", true)
+	cands := e.candidates()
+	require.Len(t, cands, 1)
+
+	e.rec.isLeader = func() bool { return false }
+	assert.False(t, e.rec.repair(context.Background(), cands[0]))
+	assert.Empty(t, e.health("lbr22").Repairs)
+}
+
+func TestLBHealth_FailedResumeThenNewVMHeartbeatsConverge(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	ctx := context.Background()
+
+	// The old VM's evidence was suspended by SetSubnets, whose resume write then failed.
+	_, err := svc.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(rec *LBDataPathRecord) (bool, error) {
+		rec.InstanceID = "i-oldvm"
+		rec.Suspended = true
+		rec.SuspendedAt = time.Now().UTC()
+		return true, nil
+	})
+	require.NoError(t, err)
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 0, 10))
+	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	assert.False(t, rec.Suspended, "a heartbeat from the VM the LB names lifts the stale suspension")
+	assert.Equal(t, healthInst, rec.InstanceID)
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 11))
+	rec, err = svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	require.Contains(t, rec.NICs, healthMACData, "a later NIC failure is recorded and repairable")
+}
+
+func TestLBHealth_SuspensionIsNotLiftedByAHeartbeatFromAVMTheLBDoesNotName(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	ctx := context.Background()
+
+	_, err := svc.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(rec *LBDataPathRecord) (bool, error) {
+		rec.InstanceID = "i-oldvm"
+		rec.Suspended = true
+		rec.SuspendedAt = time.Now().UTC()
+		return true, nil
+	})
+	require.NoError(t, err)
+
+	healthBeat(t, svc, aws.String("i-oldvm"), nicStatus(healthMACData, false, 3, 1))
+	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	assert.True(t, rec.Suspended)
+	assert.Equal(t, "i-oldvm", rec.InstanceID)
+}
+
+func TestLBHealth_SuspensionBackstopLiftsOnlyAfterThirtyMinutes(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	ctx := context.Background()
+
+	suspend := func(age time.Duration) {
+		_, err := svc.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(rec *LBDataPathRecord) (bool, error) {
+			rec.InstanceID = healthInst
+			rec.Suspended = true
+			rec.SuspendedAt = time.Now().UTC().Add(-age)
+			return true, nil
+		})
+		require.NoError(t, err)
+	}
+
+	suspend(29 * time.Minute)
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
+	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	assert.True(t, rec.Suspended, "inside the backstop the suspension holds")
+
+	suspend(31 * time.Minute)
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 2))
+	rec, err = svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	assert.False(t, rec.Suspended, "past the backstop the current VM's heartbeat lifts it")
+	assert.Contains(t, rec.NICs, healthMACData)
+}
+
+func TestLBHealth_EstablishedFailureSurvivesAnAgentRestart(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	ctx := context.Background()
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
+	first, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	since := first.NICs[healthMACData].FailingSince
+
+	// A restarted agent reports healthy until it has missed three probes again.
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 1, 2))
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 2, 3))
+	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	require.Contains(t, rec.NICs, healthMACData)
+	assert.True(t, since.Equal(rec.NICs[healthMACData].FailingSince), "FailingSince is kept")
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 0, 4))
+	rec, err = svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	assert.Empty(t, rec.NICs, "only a clean probe clears the failure")
+}
+
+func TestLBHealth_OldVMHeartbeatPausedAcrossPublicationLeavesTheNewBinding(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	ctx := context.Background()
+
+	// SetSubnets has already published the new VM and rebound the evidence.
+	lb.InstanceID = "i-newvm"
+	require.NoError(t, svc.store.PutLoadBalancer(ctx, lb))
+	require.NoError(t, svc.store.ResumeLBHealth(ctx, lb.LoadBalancerID, "i-newvm"))
+
+	// The old VM's heartbeat read the LB before that and runs now.
+	stale := *lb
+	stale.InstanceID = healthInst
+	svc.recordDataPathEvidence(ctx, &stale, &LBAgentHeartbeatInput{
+		LBID:       aws.String(lb.LoadBalancerID),
+		InstanceID: aws.String(healthInst),
+		NICs:       []*LBAgentNICStatus{nicStatus(healthMACData, false, 5, 1)},
+	})
+
+	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	require.NotNil(t, rec)
+	assert.Equal(t, "i-newvm", rec.InstanceID)
+	assert.Empty(t, rec.NICs, "the old VM's failure must not land on the new binding")
+}