fix: say RemoveSystemImage deletes the UEFI store before config.json

The instance prune, the reaper after a terminate of an imaged instance, and
the reaper after a failed launch seeded from an image each leave the
image's store object in place.

.github/scripts/go-build-cache.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Trims GOCACHE to the entries one CI run used. Go refreshes an entry's mtime
+# on use only when it is over an hour old, so "mark" backdates every restored
+# entry first and "trim" drops whatever this run neither read nor wrote.
+set -euo pipefail
+
+cache="$(go env GOCACHE)"
+mark="${RUNNER_TEMP:?}/go-build-cache.mark"
+
+# stale prints each entry this run did not touch and no touched action entry
+# names. Go skips rewriting an output it already holds, so a fresh action
+# entry can point at a backdated output.
+stale() {
+  local keep="${RUNNER_TEMP}/go-build-cache.keep"
+  find "$cache" -mindepth 2 -maxdepth 2 -name '*-a' -newer "$mark" -exec cat {} + |
+    awk '{ print "/" $3 "-d" }' >"$keep"
+  find "$cache" -mindepth 2 -maxdepth 2 ! -newer "$mark" | { grep -vFf "$keep" || true; }
+}
+
+case "${1:-}" in
+mark)
+  mkdir -p "$cache"
+  find "$cache" -mindepth 2 -maxdepth 2 -exec touch -m -d '2 hours ago' {} +
+  touch "$mark"
+  ;;
+stale | trim)
+  if [[ ! -f "$mark" ]]; then
+    echo "go-build-cache: no mark, leaving the cache untrimmed" >&2
+    exit 0
+  fi
+  if [[ "$1" == stale ]]; then
+    stale
+    exit 0
+  fi
+  before=$(du -sm "$cache" | cut -f1)
+  # An entry is a file, or a directory holding one cached executable whose
+  # mtime Go refreshes in place of the binary's, so it goes as a unit.
+  stale | xargs -r rm -rf --
+  echo "trimmed ${before}MB -> $(du -sm "$cache" | cut -f1)MB"
+  ;;
+*)
+  echo "usage: $0 mark|stale|trim" >&2
+  exit 2
+  ;;
+esac

.github/workflows/e2e-nightly.yml

@@ -6,11 +6,32 @@ name: E2E Nightly
 # `main` and `dev` as separate nightlies, and it would always execute `main`'s
 # copy of this file — leaving a fix to the workflow itself untestable until it
 # had already shipped. A dispatch takes both the file and the source from its ref.
+#
+# Cells (matrix cells are defined in .github/e2e-cells.json):
+#
+#   Single node                       Three nodes
+#    1  Debian 13, DHCP                 8  Debian 13, DHCP
+#    2  Ubuntu 26.04, static            9  Ubuntu 26.04, static
+#    3  Ubuntu 26.04, DHCP             10  Ubuntu 26.04, DHCP
+#   17  tofu examples                  29  storage fault injection
+#   18  reboot                         30  routed NAT
+#   19  routed NAT                     31  instance recovery
+#   25  eks                            32  load balancer recovery
+#   26  ecs                            33  KV bucket quorum under node loss
+#   27  rds                            37  Oracle Linux 10, DHCP
+#   34  Oracle Linux 10, DHCP          38  Oracle Linux 10, routed NAT
+#   35  Oracle Linux 10, reboot
+#   36  Oracle Linux 10, routed NAT
+#
+#   Standalone jobs
+#   20  bare metal (PXE)               23  predastore throughput
+#   21  ISO, single node               24  predastore fault injection
+#   22  ISO, three nodes               28  guest disk performance (bare metal)
 on:
   workflow_dispatch:
     inputs:
       cells:
-        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 34-36 Oracle Linux single node (basic, reboot, routed NAT), 37-38 Oracle Linux on three nodes (multi-node subset with IPsec, routed NAT), 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes"
+        description: "Comma-separated cell numbers, no spaces, e.g. 1,25,27 (blank = all). The cell list is at the top of .github/workflows/e2e-nightly.yml."
         required: false
 
 permissions:

.github/workflows/e2e.yml

@@ -18,7 +18,7 @@ on:
       suites:
         description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam e2e-rds')"
         required: false
-        default: "e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam"
+        default: "e2e-single e2e-multinode e2e-cert e2e-iam"
       test_run:
         description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite."
         required: false
@@ -31,7 +31,9 @@ permissions:
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
-  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam e2e-partialblock' }}
+  # lb is out of the push default: it runs in the nightly multi cells and on
+  # dispatch (suites=e2e-lb), which keeps the multi job to multinode and cert.
+  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-iam e2e-partialblock' }}
   TEST_RUN: ${{ inputs.test_run || '' }}
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

.github/workflows/spinifex.yml

@@ -105,6 +105,13 @@ jobs:
           key: gobuild-${{ github.job }}-${{ github.run_id }}
           restore-keys: gobuild-${{ github.job }}-
 
+      # Go refreshes an entry's mtime on use only when it is over an hour old, so
+      # restored entries are backdated past that before any Go command runs and
+      # the trim below can tell this run's working set from the last run's.
+      - name: Mark Go build cache
+        if: ${{ github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' }}
+        run: spinifex/.github/scripts/go-build-cache.sh mark
+
       - name: Setup Go Workspace
         run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
@@ -137,19 +144,12 @@ jobs:
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
 
-      # Compiled output stays valid whatever the tests said, and skipping the
-      # save on a red run would leave the next one rebuilding it, so this is
-      # gated on the branch rather than on success.
-      # Go bumps an entry's mtime when a run uses it, so this run's working set
-      # survives and objects superseded by later commits do not. Runs after the
-      # work, so it costs this job nothing and only shrinks what the next reads.
+      # Keeps what this run read or wrote and drops the rest, so the saved entry
+      # is one run's working set however often dev is pushed. A red run may have
+      # stopped early and used only part of the set, so it saves untrimmed.
       - name: Trim Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        run: |
-          C="$(go env GOCACHE)"
-          before=$(du -sm "$C" | cut -f1)
-          find "$C" -type f -mmin +240 -delete
-          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
+        if: ${{ success() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: spinifex/.github/scripts/go-build-cache.sh trim
 
       - name: Save Go build cache
         if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
@@ -235,22 +235,25 @@ jobs:
           key: gobuild-${{ github.job }}-${{ github.run_id }}
           restore-keys: gobuild-${{ github.job }}-
 
+      # Go refreshes an entry's mtime on use only when it is over an hour old, so
+      # restored entries are backdated past that before any Go command runs and
+      # the trim below can tell this run's working set from the last run's.
+      - name: Mark Go build cache
+        if: ${{ github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' }}
+        run: spinifex/.github/scripts/go-build-cache.sh mark
+
       - name: Setup Go Workspace
         run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Tests with Race Detector
         run: make -C spinifex test-race GOTESTFLAGS=-count=1
 
-      # Go bumps an entry's mtime when a run uses it, so this run's working set
-      # survives and objects superseded by later commits do not. Runs after the
-      # work, so it costs this job nothing and only shrinks what the next reads.
+      # Keeps what this run read or wrote and drops the rest, so the saved entry
+      # is one run's working set however often dev is pushed. A red run may have
+      # stopped early and used only part of the set, so it saves untrimmed.
       - name: Trim Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        run: |
-          C="$(go env GOCACHE)"
-          before=$(du -sm "$C" | cut -f1)
-          find "$C" -type f -mmin +240 -delete
-          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
+        if: ${{ success() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: spinifex/.github/scripts/go-build-cache.sh trim
 
       - name: Save Go build cache
         if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
@@ -337,6 +340,13 @@ jobs:
           key: gobuild-${{ github.job }}-${{ github.run_id }}
           restore-keys: gobuild-${{ github.job }}-
 
+      # Go refreshes an entry's mtime on use only when it is over an hour old, so
+      # restored entries are backdated past that before any Go command runs and
+      # the trim below can tell this run's working set from the last run's.
+      - name: Mark Go build cache
+        if: ${{ github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' }}
+        run: spinifex/.github/scripts/go-build-cache.sh mark
+
       - name: Setup Go Workspace
         run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
@@ -353,16 +363,12 @@ jobs:
         if: ${{ !cancelled() }}
         run: make -C spinifex test-integration
 
-      # Go bumps an entry's mtime when a run uses it, so this run's working set
-      # survives and objects superseded by later commits do not. Runs after the
-      # work, so it costs this job nothing and only shrinks what the next reads.
+      # Keeps what this run read or wrote and drops the rest, so the saved entry
+      # is one run's working set however often dev is pushed. A red run may have
+      # stopped early and used only part of the set, so it saves untrimmed.
       - name: Trim Go build cache
-        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
-        run: |
-          C="$(go env GOCACHE)"
-          before=$(du -sm "$C" | cut -f1)
-          find "$C" -type f -mmin +240 -delete
-          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
+        if: ${{ success() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: spinifex/.github/scripts/go-build-cache.sh trim
 
       - name: Save Go build cache
         if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}

Makefile

@@ -309,7 +309,8 @@ test-build-scripts:
 	done
 	$(_Q)shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh \
 		.github/scripts/wait-for-memory.sh \
-		.github/scripts/mulga-design-check.sh
+		.github/scripts/mulga-design-check.sh \
+		.github/scripts/go-build-cache.sh
 	@$(_HDR) "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
 	$(_Q)shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
 	$(_Q)shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)

cmd/lambda-image-agent-tag/main.go

@@ -159,6 +159,9 @@ func parseEntry(raw []byte, off int) (entry, int, error) {
 	}
 	nameEnd := off + newcHeader + int(nameSize)
 	dataOff := align4(nameEnd)
+	if dataOff > len(raw) || size > uint64(len(raw)-dataOff) {
+		return entry{}, 0, errors.New("truncated entry")
+	}
 	dataEnd := dataOff + int(size)
 	if dataEnd > len(raw) || align4(dataEnd) > len(raw) {
 		return entry{}, 0, errors.New("truncated entry")

contracts/cluster/v1/heartbeat.go

@@ -1,6 +1,9 @@
 package clusterv1
 
-import "time"
+import (
+	"encoding/json"
+	"time"
+)
 
 // ClusterStateBucket is the KV bucket holding each daemon's heartbeat alongside
 // other cluster-wide state.
@@ -11,6 +14,27 @@ func HeartbeatKey(node string) string {
 	return "heartbeat." + node
 }
 
+// LivenessWindow is how recently a node must have heartbeated for a reader
+// deciding on a takeover to count it live. It is six heartbeats, compared
+// across node clocks, where NTP skew is small against it.
+const LivenessWindow = 60 * time.Second
+
+// LiveAt reports whether a heartbeat written at written still vouches for its
+// node at now: it does while it is under LivenessWindow old.
+func LiveAt(written, now time.Time) bool {
+	return now.Sub(written) < LivenessWindow
+}
+
+// HeartbeatWritten dates a stored heartbeat value. One that cannot be decoded
+// or dated vouches for nothing, which is an answer rather than a fault.
+func HeartbeatWritten(data []byte) (time.Time, bool) {
+	var beat HeartbeatTimestamp
+	if json.Unmarshal(data, &beat) != nil {
+		return time.Time{}, false
+	}
+	return beat.Time()
+}
+
 // HeartbeatTimestamp is the read side of a heartbeat for node liveness. It
 // decodes only the timestamp, so a reader does not depend on the capacity
 // figures the daemon writes beside it.

contracts/cluster/v1/heartbeat_test.go

@@ -33,3 +33,23 @@ func TestHeartbeatTimestamp_UndatedBeatVouchesForNothing(t *testing.T) {
 		assert.False(t, ok, "timestamp %q", ts)
 	}
 }
+
+// A beat is live strictly under the window, so one exactly the window old is
+// the first a takeover may treat as stopped.
+func TestLiveAt_IsStrictlyUnderTheWindow(t *testing.T) {
+	now := time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)
+	assert.True(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow+time.Nanosecond), now))
+	assert.False(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow), now))
+	assert.True(t, clusterv1.LiveAt(now.Add(time.Second), now), "a beat from a clock running ahead is live")
+}
+
+func TestHeartbeatWritten_DatesAStoredBeatAndRefusesAnUndecodableOne(t *testing.T) {
+	written, ok := clusterv1.HeartbeatWritten([]byte(`{"node":"node-a","timestamp":"2026-10-30T01:02:03Z"}`))
+	require.True(t, ok)
+	assert.True(t, written.Equal(time.Date(2026, 10, 30, 1, 2, 3, 0, time.UTC)))
+
+	for _, raw := range []string{"not json", `{"node":"node-a"}`} {
+		_, ok := clusterv1.HeartbeatWritten([]byte(raw))
+		assert.False(t, ok, raw)
+	}
+}

docs/DESIGN.md

@@ -111,7 +111,7 @@ Daemons (`spinifex/runtime/daemon/daemon.go`) subscribe to NATS topics and handl
 When a daemon handles `RunInstances`:
 
 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
-2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
+2. **Volume Generation**: Creates the boot volume through the EBS provider and, for a UEFI guest, seeds its variable store as an object under `uefi/<instance-id>/` with a store record in `spinifex-uefi-stores` that points at it
 3. **Volume Mount**: Publishes each volume through the EBS provider contract (`PublishVolume` on `ebs.provider.v1.{node}.mount`), receives its export
 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management

scripts/README.md

@@ -98,7 +98,7 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o
 | `build-system-image.sh` | Builds a minimal system AMI (Alpine or Ubuntu) from a manifest. All customisation happens inside the libguestfs appliance, so the build touches no host block device, no host mount, and needs no sudo. `--import` also registers it as an AMI. |
 | `publish-system-image.sh` | Converts a built raw image to compressed qcow2, writes the `.sha256` sidecar the catalog verifier expects, and uploads both to the R2 bucket behind `iso.mulgadc.com/system-ami/`. |
 | `build-microvm-image.sh` | Builds the minimal Alpine kernel + initramfs used by the QEMU microvm machine type and the lb-agent, carrying haproxy (ALB) and nginx-stream (NLB) data planes. |
-| `build-qboot.sh` | Builds qboot, the firmware the daemon passes to every microvm with `-bios`, from a pinned upstream commit in a digest-pinned Debian container. Fails unless the output matches the recorded sha256, which is also Debian 13's `qboot.rom`. Writes `qboot.rom` and its GPL-2.0 licence beside the microvm kernel, and the matching source tarball, which releases publish, under `build/qboot/`. `make build-microvm-image` runs it. |
+| `build-qboot.sh` | Builds qboot, the firmware the daemon passes to every microvm with `-bios`, from a pinned upstream commit in a digest-pinned Debian container. Fails unless the output matches the recorded sha256, which is also Debian 13's `qboot.rom`. A host whose `/usr/share/qemu/qboot.rom` (`qemu-system-data`) has that hash installs it and skips the container; any other host compiles. Writes `qboot.rom` and its GPL-2.0 licence beside the microvm kernel, and the matching source tarball, which releases publish, under `build/qboot/`. `make build-microvm-image` runs it. |
 | `mkosi-build.sh` | Runs an mkosi system-image build inside the pinned builder container, so a runner needs only Docker and the toolchain cannot drift. `--image` names an output and expands to the right profile list. |
 | `mkosi-publish.sh` | Publishes an mkosi-built image to R2 from inside the same pinned container — qcow2 conversion, `.sha256` sidecar, and upload. |
 | `mkosi-common.sh` | Shared plumbing sourced by both mkosi scripts. Builds the toolchain image once, with the builder account matched to the invoking user so bind-mounted output is not written back as root. |

scripts/build-qboot.sh

@@ -9,7 +9,8 @@
 #
 # The build runs in a digest-pinned Debian container, so its output is
 # reproducible and is checked against a recorded hash. That hash is also the
-# sha256 of Debian 13's /usr/share/qemu/qboot.rom, which is built from the same commit.
+# sha256 of Debian 13's /usr/share/qemu/qboot.rom, which is built from the same
+# commit, so a host whose copy matches installs it and skips the container.
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
@@ -20,18 +21,13 @@ QBOOT_REPO="${QBOOT_REPO:-https://gitlab.com/qemu-project/qboot.git}"
 QBOOT_COMMIT="8ca302e86d685fa05b16e2b208888243da319941"
 QBOOT_SHA256="5c9c4b12c4438e3cf11d087263afe6519c8c05dd5896727b8b169b8f5dbf9877"
 BUILD_IMAGE="debian:trixie@sha256:4c25275446ca371766b8b093c696782eabce06263fccacb8d5d813b3dc44b308"
+# Debian 13's qemu-system-data ships this exact ROM, so a host that has it
+# skips the container. QBOOT_HOST_ROM moves the path for tests.
+QBOOT_HOST_ROM="${QBOOT_HOST_ROM:-/usr/share/qemu/qboot.rom}"
 
 log() { echo "[build-qboot] $*"; }
 fail() { echo "ERROR: $*" >&2; exit 1; }
 
-CONTAINER_TOOL=""
-for tool in docker podman; do
-    if command -v "$tool" >/dev/null 2>&1 && "$tool" info >/dev/null 2>&1; then
-        CONTAINER_TOOL=$tool
-        break
-    fi
-done
-[ -n "$CONTAINER_TOOL" ] || fail "qboot builds in a container; install docker or podman"
 command -v git >/dev/null 2>&1 || fail "required tool not found: git"
 
 WORK=$(mktemp -d)
@@ -48,19 +44,42 @@ mkdir -p "$SRC_DIR"
 src_tarball="$SRC_DIR/qboot-${QBOOT_COMMIT}.tar.gz"
 git -C "$WORK/src" archive --format=tar.gz --prefix="qboot-${QBOOT_COMMIT}/" -o "$src_tarball" "$QBOOT_COMMIT"
 
-log "building in $BUILD_IMAGE with $CONTAINER_TOOL"
-# The firmware comes back on stdout, so nothing the container writes is left
-# root-owned on the host.
-"$CONTAINER_TOOL" run --rm --platform linux/amd64 -v "$WORK/src:/src:ro" \
-    -e DEBIAN_FRONTEND=noninteractive "$BUILD_IMAGE" sh -euc '
-        {
-            apt-get update -qq
-            apt-get install -y -qq --no-install-recommends gcc libc6-dev meson ninja-build binutils
-            meson setup /tmp/build /src
-            ninja -C /tmp/build
-        } >/dev/null
-        cat /tmp/build/bios.bin
-    ' > "$WORK/qboot.rom"
+build_in_container() {
+    local CONTAINER_TOOL=""
+    for tool in docker podman; do
+        if command -v "$tool" >/dev/null 2>&1 && "$tool" info >/dev/null 2>&1; then
+            CONTAINER_TOOL=$tool
+            break
+        fi
+    done
+    [ -n "$CONTAINER_TOOL" ] || fail "qboot builds in a container; install docker or podman"
+    log "building in $BUILD_IMAGE with $CONTAINER_TOOL"
+    # The firmware comes back on stdout, so nothing the container writes is left
+    # root-owned on the host.
+    "$CONTAINER_TOOL" run --rm --platform linux/amd64 -v "$WORK/src:/src:ro" \
+        -e DEBIAN_FRONTEND=noninteractive "$BUILD_IMAGE" sh -euc '
+            {
+                apt-get update -qq
+                apt-get install -y -qq --no-install-recommends gcc libc6-dev meson ninja-build binutils
+                meson setup /tmp/build /src
+                ninja -C /tmp/build
+            } >/dev/null
+            cat /tmp/build/bios.bin
+        ' > "$WORK/qboot.rom"
+}
+
+host_sha=""
+[ -r "$QBOOT_HOST_ROM" ] && host_sha=$(sha256sum "$QBOOT_HOST_ROM" | cut -d' ' -f1)
+if [ "$host_sha" = "$QBOOT_SHA256" ]; then
+    log "using host ROM $QBOOT_HOST_ROM (sha256 matches)"
+    cp "$QBOOT_HOST_ROM" "$WORK/qboot.rom"
+elif [ -n "$host_sha" ]; then
+    log "host ROM $QBOOT_HOST_ROM has sha256 $host_sha, not $QBOOT_SHA256; building"
+    build_in_container
+else
+    log "no host ROM at $QBOOT_HOST_ROM; building"
+    build_in_container
+fi
 
 got_sha=$(sha256sum "$WORK/qboot.rom" | cut -d' ' -f1)
 if [ "$got_sha" != "$QBOOT_SHA256" ]; then

scripts/build-qboot_test.sh

@@ -0,0 +1,88 @@
+#!/bin/sh
+# build-qboot_test.sh — a host ROM matching the recorded hash is installed
+# without the container; any other host ROM, or none, builds in it, and a
+# container ROM with the wrong hash still fails.
+set -eu
+
+SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+echo "-- build-qboot.sh host ROM"
+
+COMMIT=$(sed -n 's/^QBOOT_COMMIT="\(.*\)"$/\1/p' "$SCRIPTS_DIR/build-qboot.sh")
+printf 'pinned rom\n' > "$TMP/good.rom"
+printf 'other rom\n' > "$TMP/other.rom"
+GOOD_SHA=$(sha256sum "$TMP/good.rom" | cut -d' ' -f1)
+
+# git and docker stubs. git fakes a checkout of the pinned commit; docker
+# prints $STUB_ROM as the firmware and records that it ran.
+mkdir -p "$TMP/bin"
+cat > "$TMP/bin/git" <<STUB
+#!/bin/sh
+case "\$1" in
+init) mkdir -p "\$3"; printf 'GPL-2.0\n' > "\$3/LICENSE"; exit 0 ;;
+esac
+shift 2
+case "\$1" in
+rev-parse) echo "$COMMIT" ;;
+archive) while [ "\$1" != "-o" ]; do shift; done; printf 'source\n' > "\$2" ;;
+esac
+STUB
+cat > "$TMP/bin/docker" <<'STUB'
+#!/bin/sh
+[ "$1" = info ] && exit 0
+echo ran >> "$DOCKER_LOG"
+cat "$STUB_ROM"
+STUB
+chmod +x "$TMP/bin/git" "$TMP/bin/docker"
+
+# run <name> <host-rom> <container-rom>: build in a fresh fixture repo.
+run() {
+    repo="$TMP/$1"
+    mkdir -p "$repo/scripts"
+    sed "s/^QBOOT_SHA256=.*/QBOOT_SHA256=\"$GOOD_SHA\"/" "$SCRIPTS_DIR/build-qboot.sh" > "$repo/scripts/build-qboot.sh"
+    : > "$TMP/$1.docker"
+    PATH="$TMP/bin:$PATH" QBOOT_HOST_ROM="$2" STUB_ROM="$3" DOCKER_LOG="$TMP/$1.docker" \
+        bash "$repo/scripts/build-qboot.sh" > "$TMP/$1.log" 2>&1
+}
+
+# outputs <name>: the ROM is the pinned one and the licence and source exist.
+outputs() {
+    cmp -s "$TMP/$1/build/microvm/qboot.rom" "$TMP/good.rom" &&
+        [ -s "$TMP/$1/build/microvm/qboot.LICENSE" ] &&
+        [ -s "$TMP/$1/build/qboot/qboot-$COMMIT.tar.gz" ]
+}
+
+if run match "$TMP/good.rom" "$TMP/other.rom" && outputs match && [ ! -s "$TMP/match.docker" ]; then
+    pass "matching host ROM installed, container not run"
+else
+    fail "matching host ROM: $(tr '\n' ' ' < "$TMP/match.log")"
+fi
+
+if run mismatch "$TMP/other.rom" "$TMP/good.rom" && outputs mismatch && [ -s "$TMP/mismatch.docker" ] &&
+    grep -q 'has sha256' "$TMP/mismatch.log"; then
+    pass "mismatched host ROM builds in the container"
+else
+    fail "mismatched host ROM: $(tr '\n' ' ' < "$TMP/mismatch.log")"
+fi
+
+if run missing "$TMP/absent.rom" "$TMP/good.rom" && outputs missing && [ -s "$TMP/missing.docker" ]; then
+    pass "missing host ROM builds in the container"
+else
+    fail "missing host ROM: $(tr '\n' ' ' < "$TMP/missing.log")"
+fi
+
+if run badbuild "$TMP/absent.rom" "$TMP/other.rom"; then
+    fail "container ROM with the wrong hash was accepted"
+elif [ -e "$TMP/badbuild/build/microvm/qboot.rom" ]; then
+    fail "container ROM with the wrong hash was installed"
+else
+    pass "container ROM with the wrong hash fails and installs nothing"
+fi
+
+[ "$fails" -eq 0 ] || { echo "$fails failure(s)"; exit 1; }

spinifex/domains/acm/store.go

@@ -14,7 +14,6 @@ import (
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -184,7 +183,7 @@ func NewStore(ctx context.Context, nc *nats.Conn, masterKey []byte) (*Store, err
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketACM, KVBucketACMVersion)
+	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketACM, KVBucketACMVersion)
 	if err != nil {
 		return nil, err
 	}

spinifex/domains/admission/quota/bedrock.go

@@ -10,7 +10,7 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"golang.org/x/time/rate"
 )
@@ -171,7 +171,7 @@ func (s *Service) RunBedrockRPMSync(ctx context.Context, js jetstream.JetStream)
 	if s == nil || !s.limits.RequestsPerMinuteEnabled {
 		return
 	}
-	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketBedrockRPMUsage, 1)
+	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketBedrockRPMUsage, 1)
 	if err != nil {
 		slog.Warn("bedrock rpm sync: open usage bucket failed, sync disabled", "err", err)
 		return

spinifex/domains/admission/quota/override.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -187,7 +187,7 @@ func (s *Service) PutAccountQuota(ctx context.Context, accountID string, over Ov
 	over.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
 	over.UpdatedBy = updatedBy
 
-	_, err := kvutil.Update(ctx, s.overrides, accountID, kvutil.CASConfig{
+	_, err := kvstore.Update(ctx, s.overrides, accountID, kvstore.CASConfig{
 		CreateIfAbsent: true,
 		Attempts:       overrideMaxAttempts,
 		Exhausted:      func(string, int) error { return errOverridesContended },
@@ -230,7 +230,7 @@ func (s *Service) clearAccountQuota(ctx context.Context, accountID string) error
 			return nil
 		case errors.Is(err, jetstream.ErrKeyNotFound):
 			return nil
-		case kvutil.IsRevisionConflict(err):
+		case kvstore.IsRevisionConflict(err):
 			// A concurrent write landed between the read and the guarded
 			// delete; retry against whatever revision now holds.
 			continue

spinifex/domains/admission/quota/quota.go

@@ -22,10 +22,10 @@ const ReconcileInterval = 30 * time.Second
 // dimension; the live-counted dimensions need no stored state.
 const KVBucketAccountUsage = "spinifex-account-usage"
 
-// KVBucketQuotaReconcile is the leader-lock bucket for the gateway quota
-// reconcile. It is deliberately distinct from vpcd's network-reconcile lock so
-// the two independent loops never share one mutex and block each other.
-const KVBucketQuotaReconcile = "spinifex-quota-reconcile"
+// KeyReconcile is the sync-bucket leader lease for the gateway quota reconcile.
+// It is deliberately distinct from vpcd's network-reconcile lease so the two
+// independent loops never share one mutex and block each other.
+const KeyReconcile = "quota.reconcile"
 
 // KVBucketAccountQuota holds the per-account limit overrides, keyed by
 // accountID. It is separate from the accounts bucket so a quota write never

spinifex/domains/admission/quota/seal.go

@@ -8,7 +8,7 @@ import (
 
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -73,7 +73,7 @@ func (s *Service) sealBucket(ctx context.Context) (jetstream.KeyValue, error) {
 		slog.ErrorContext(ctx, "Quota override seal has no JetStream client wired to open the override bucket")
 		return nil, errors.New(awserrors.ErrorInternalError)
 	}
-	kv, err := kvutil.GetOrCreateBucket(ctx, s.js, KVBucketAccountQuota, 1)
+	kv, err := kvstore.GetOrCreateBucket(ctx, s.js, KVBucketAccountQuota, 1)
 	if err != nil {
 		slog.ErrorContext(ctx, "Failed to open quota override bucket for seal", "err", err)
 		return nil, errors.New(awserrors.ErrorInternalError)
@@ -136,7 +136,7 @@ func (s *Service) SealAccountOverrides(ctx context.Context, accountID string) (*
 		switch {
 		case writeErr == nil:
 			return &OverrideSeal{Revision: newRev}, nil
-		case kvutil.IsRevisionConflict(writeErr):
+		case kvstore.IsRevisionConflict(writeErr):
 			continue
 		default:
 			slog.ErrorContext(ctx, "Failed to write sealed quota override record", "accountID", accountID, "err", writeErr)

spinifex/domains/dns/delegation.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	nsconfig "github.com/mulgadc/northstar/pkg/config"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 )
@@ -48,7 +48,7 @@ func (w *Writer) PlatformDelegation(name string) (servers []string, zone string,
 // Delegate writes, or with remove withdraws, an NS set in the base zone that
 // delegates each name to this cluster's nameservers. It takes the zone-writer
 // election first, so it never races the reconcile's own writes to the zone.
-func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, names []string, remove bool) (*ChangeResult, error) {
+func (w *Writer) Delegate(ctx context.Context, lease *kvsync.Lease, names []string, remove bool) (*ChangeResult, error) {
 	if !w.Enabled() {
 		return nil, errors.New("northstar is not configured on this node")
 	}
@@ -77,7 +77,7 @@ func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, nam
 	if len(batch.Changes) == 0 {
 		return &ChangeResult{}, nil
 	}
-	release, err := w.awaitWriterElection(ctx, nc, holder)
+	release, err := w.awaitWriterElection(ctx, lease)
 	if err != nil {
 		return nil, err
 	}
@@ -85,19 +85,19 @@ func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, nam
 	return w.ApplyBatch(batch)
 }
 
-func (w *Writer) awaitWriterElection(ctx context.Context, nc *nats.Conn, holder string) (func(), error) {
-	if nc == nil {
+func (w *Writer) awaitWriterElection(ctx context.Context, lease *kvsync.Lease) (func(), error) {
+	if lease == nil {
 		return func() {}, nil
 	}
 	tick := time.NewTicker(delegateLockPoll)
 	defer tick.Stop()
 	for {
-		if release, ok := reconcilelock.AcquireLeader(ctx, nc, KVBucketDNSReconcile, holder); ok {
+		if release, ok := reconcilelock.AcquireLeader(ctx, lease); ok {
 			return release, nil
 		}
 		select {
 		case <-ctx.Done():
-			return nil, fmt.Errorf("another node is writing the zone: %w", ctx.Err())
+			return nil, fmt.Errorf("another writer has the zone: %w", ctx.Err())
 		case <-tick.C:
 		}
 	}

spinifex/domains/dns/delegation_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
+	_, err := w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
 	require.NoError(t, err)
 
 	for _, name := range []string{"a.r53e2e.spx3.net.", "b.r53e2e.spx3.net"} {
@@ -29,7 +29,7 @@ func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 	assert.True(t, ok, "a name below the base domain is decided by the base zone")
 	assert.Empty(t, servers)
 
-	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
+	_, err = w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, true)
 	require.NoError(t, err)
 	servers, _, _, err = w.PlatformDelegation("a.r53e2e.spx3.net")
 	require.NoError(t, err)
@@ -43,11 +43,11 @@ func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 // servers written while the cluster published a larger set.
 func TestDelegateRemoveWithdrawsServersFromAnEarlierSet(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, false)
+	_, err := w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, false)
 	require.NoError(t, err)
 
 	w.nameservers = w.nameservers[:1]
-	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
+	_, err = w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, true)
 	require.NoError(t, err)
 	servers, _, _, err := w.PlatformDelegation("a.r53e2e.spx3.net")
 	require.NoError(t, err)
@@ -66,7 +66,7 @@ func TestPlatformDelegationIgnoresNamesOutsideTheBaseDomain(t *testing.T) {
 func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
 	for _, name := range []string{"ns1.spx3.net", "x.elb.spx3.net", "compute.spx3.net", "example.com", "spx3.net"} {
-		_, err := w.Delegate(t.Context(), nil, "node1", []string{"ok.spx3.net", name}, false)
+		_, err := w.Delegate(t.Context(), nil, []string{"ok.spx3.net", name}, false)
 		require.Error(t, err, name)
 	}
 	servers, _, _, err := w.PlatformDelegation("ok.spx3.net")
@@ -76,22 +76,45 @@ func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
 
 func TestDelegateWaitsForTheZoneWriterElection(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	prev := delegateLockPoll
 	delegateLockPoll = 10 * time.Millisecond
 	t.Cleanup(func() { delegateLockPoll = prev })
 
-	release, ok := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "node2")
+	release, ok := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "node2"))
 	require.True(t, ok)
 	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
 	defer cancel()
-	_, err := w.Delegate(ctx, nc, "node1", []string{"a.spx3.net"}, false)
-	require.ErrorContains(t, err, "another node is writing the zone")
+	node1 := dnsLease(js, "node1")
+	_, err := w.Delegate(ctx, node1, []string{"a.spx3.net"}, false)
+	require.ErrorContains(t, err, "another writer has the zone")
 
 	release()
-	_, err = w.Delegate(t.Context(), nc, "node1", []string{"a.spx3.net"}, false)
+	_, err = w.Delegate(t.Context(), node1, []string{"a.spx3.net"}, false)
 	require.NoError(t, err)
 	servers, _, _, err := w.PlatformDelegation("a.spx3.net")
 	require.NoError(t, err)
 	assert.Len(t, servers, 2)
 }
+
+// The reconcile pass and a delegation on one node share the DNS lease, so a
+// delegation waits for this node's own pass as it would for another node's.
+func TestDelegateWaitsForThisNodesOwnReconcilePass(t *testing.T) {
+	w, _ := newMultiNodeTestWriter(t)
+	_, _, js := testutil.StartTestJetStream(t)
+	prev := delegateLockPoll
+	delegateLockPoll = 10 * time.Millisecond
+	t.Cleanup(func() { delegateLockPoll = prev })
+
+	shared := dnsLease(js, "node1")
+	release, ok := reconcilelock.AcquireLeader(t.Context(), shared)
+	require.True(t, ok, "the reconcile pass holds the lease")
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+	_, err := w.Delegate(ctx, shared, []string{"a.spx3.net"}, false)
+	require.ErrorContains(t, err, "another writer has the zone")
+
+	release()
+	_, err = w.Delegate(t.Context(), shared, []string{"a.spx3.net"}, false)
+	require.NoError(t, err)
+}

spinifex/domains/dns/reconcile.go

@@ -11,7 +11,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 )
 
 // DefaultReconcileInterval is how often the drift backstop rebuilds managed
@@ -33,6 +33,12 @@ const DefaultReconcileInterval = reconciler.DefaultResync
 // costs a cycle for nothing.
 const DefaultComputeTimeout = 2 * time.Minute
 
+// NewReconcileLease returns the zone writer's election lease, shared by the
+// drift reconcile and Delegate so the two never write the zone together.
+func NewReconcileLease(store kvsync.KVStore, holder string) *kvsync.Lease {
+	return reconcilelock.NewLeaderLease(store, KeyReconcile, holder)
+}
+
 // DesiredFunc returns the full desired managed record set built from the live
 // resource inventory across all tenants. The daemon supplies it by enumerating
 // instances, load balancers, and EKS clusters.
@@ -102,7 +108,8 @@ type Reconciler struct {
 	// lands in. Distinct from both baseDomain and internalDomain, so pruning it
 	// needs its own read the same way internalDomain does for EC2.
 	serviceZone string
-	nc          *nats.Conn
+	// lease elects the one node that writes the zones; nil skips the election.
+	lease *kvsync.Lease
 	// writer applies the converged batch. The reconcile owns the zone object on
 	// the node that won the election, so the batch never leaves this process.
 	writer   *Writer
@@ -111,7 +118,6 @@ type Reconciler struct {
 	// computeTimeout bounds how long one pass spends building the desired set.
 	// A field rather than a constant so a test can shrink it.
 	computeTimeout time.Duration
-	holder         string
 	sources        []reconciler.Source
 }
 
@@ -121,18 +127,15 @@ type Reconciler struct {
 // changes should wake the loop. They are optional: with none supplied the loop
 // falls back to the interval alone, which is the behaviour that predates the
 // watch.
-func NewReconciler(cfg *config.Config, cluster *config.ClusterConfig, nc *nats.Conn, writer *Writer, desired DesiredFunc, sources ...reconciler.Source) *Reconciler {
+func NewReconciler(cfg *config.Config, cluster *config.ClusterConfig, lease *kvsync.Lease, writer *Writer, desired DesiredFunc, sources ...reconciler.Source) *Reconciler {
 	r := &Reconciler{
-		nc:             nc,
+		lease:          lease,
 		writer:         writer,
 		desired:        desired,
 		interval:       DefaultReconcileInterval,
 		computeTimeout: DefaultComputeTimeout,
 		sources:        sources,
 	}
-	if cfg != nil {
-		r.holder = cfg.Node
-	}
 	zoneCfg, ok := zoneS3Config(cfg)
 	if !ok || desired == nil || !writer.Enabled() {
 		return r
@@ -181,8 +184,8 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) {
 	if !r.enabled {
 		return
 	}
-	if r.nc != nil {
-		release, elected := reconcilelock.AcquireLeader(ctx, r.nc, KVBucketDNSReconcile, r.holder)
+	if r.lease != nil {
+		release, elected := reconcilelock.AcquireLeader(ctx, r.lease)
 		if !elected {
 			return
 		}

spinifex/domains/dns/reconcile_election_test.go

@@ -8,35 +8,42 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/testkit"
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// electingReconcilerFor is reconcilerFor with a live connection, so a pass runs
-// the election instead of skipping it.
-func electingReconcilerFor(w *Writer, nc *nats.Conn, holder string, desired func() DesiredSet) *Reconciler {
+// electingReconcilerFor is reconcilerFor with a lease, so a pass runs the
+// election instead of skipping it.
+func electingReconcilerFor(w *Writer, lease *kvsync.Lease, desired func() DesiredSet) *Reconciler {
 	r := reconcilerFor(w, desired)
-	r.nc = nc
-	r.holder = holder
+	r.lease = lease
 	return r
 }
 
+// dnsLease returns holder's DNS lease on a store of its own, standing in for
+// a separate process.
+func dnsLease(js jetstream.JetStream, holder string) *kvsync.Lease {
+	return NewReconcileLease(kvstore.OpenSyncBucket(js), holder)
+}
+
 // The zone lock is gone, and what replaces it is that a zone has one writer:
 // the node holding the reconcile lease. A node that loses the election must
 // therefore not touch the object at all — if it did, two nodes could be
 // read-modify-writing the same zone with nothing serialising them.
 func TestReconcileOnce_ALoserOfTheElectionDoesNotWriteTheZone(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	w, objects := newTestWriter(t)
 	before := objects[testBase+".toml"]
 	require.NotEmpty(t, before)
 
 	// A peer holds the lease for the whole pass, so this node cannot win it.
-	release, elected := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "peer")
+	release, elected := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "peer"))
 	require.True(t, elected, "the peer has to hold the lease or this test proves nothing")
 
-	r := electingReconcilerFor(w, nc, "node1", func() DesiredSet {
+	r := electingReconcilerFor(w, dnsLease(js, "node1"), func() DesiredSet {
 		return DesiredSet{Changes: []Change{upsert("lb-1.elb."+testBase, "10.200.1.9")}}
 	})
 	r.reconcileOnce(t.Context())
@@ -50,10 +57,10 @@ func TestReconcileOnce_ALoserOfTheElectionDoesNotWriteTheZone(t *testing.T) {
 // The counterpart, so the test above is not passing because the pass was broken
 // for some other reason: with the lease free, the same reconciler writes.
 func TestReconcileOnce_TheWinnerOfTheElectionWritesTheZone(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	w, objects := newTestWriter(t)
 
-	r := electingReconcilerFor(w, nc, "node1", func() DesiredSet {
+	r := electingReconcilerFor(w, dnsLease(js, "node1"), func() DesiredSet {
 		return DesiredSet{Changes: []Change{upsert("lb-1.elb."+testBase, "10.200.1.9")}}
 	})
 	r.reconcileOnce(t.Context())

spinifex/domains/dns/reconcile_test.go

@@ -224,7 +224,7 @@ func TestReconciler_RunReturnsWhenDisabled(t *testing.T) {
 // path of Run: the startup pass must fire before any watch or resync activity,
 // and cancelling ctx must still let the loop return.
 func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	endpoint, _ := fakeS3(t, "northstar")
 
 	var passes atomic.Int64
@@ -235,8 +235,7 @@ func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
 			Endpoint: endpoint, Bucket: "northstar", Region: "us-east-1",
 			AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET",
 		},
-		nc:     nc,
-		holder: "test-node",
+		lease: dnsLease(js, "test-node"),
 		desired: func() DesiredSet {
 			passes.Add(1)
 			return DesiredSet{}
@@ -268,7 +267,7 @@ func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
 // acquirable afterwards, not merely on the call returning, since it is the
 // leadership and not the pass that the rest of the cluster waits on.
 func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	endpoint, _ := fakeS3(t, "northstar")
 
 	blocked := make(chan struct{})
@@ -281,8 +280,7 @@ func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
 			Endpoint: endpoint, Bucket: "northstar", Region: "us-east-1",
 			AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET",
 		},
-		nc:             nc,
-		holder:         "stuck-node",
+		lease:          dnsLease(js, "stuck-node"),
 		computeTimeout: 100 * time.Millisecond,
 		desired: func() DesiredSet {
 			<-blocked
@@ -302,7 +300,7 @@ func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
 		t.Fatal("reconcileOnce never returned: a stuck pass still holds the lease forever")
 	}
 
-	release, elected := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "next-node")
+	release, elected := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "next-node"))
 	require.True(t, elected, "another node must be able to take the next cycle")
 	release()
 }

spinifex/domains/dns/types.go

@@ -11,9 +11,9 @@ const (
 	// PUT so every northstar instance reloads just that zone immediately, instead
 	// of waiting for the next S3 sync poll. No queue group: all servers consume.
 	SubjectZoneReload = "dns.zone.reload"
-	// KVBucketDNSReconcile elects the one node that writes the zones for a drift
-	// cycle. Its own bucket, so the election never blocks another reconcile loop.
-	KVBucketDNSReconcile = "spinifex-dns-reconcile"
+	// KeyReconcile elects the one node that writes the zones for a drift cycle.
+	// Its own key, so the election never blocks another reconcile loop.
+	KeyReconcile = "dns.reconcile"
 	// PrivateZone is the fixed AWS-parity private DNS zone (IMDS synthHostname).
 	PrivateZone = "compute.internal"
 	// DefaultTTL is applied when a change omits a TTL.

spinifex/domains/ec2/account/seal.go

@@ -8,7 +8,7 @@ import (
 
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -109,7 +109,7 @@ func (s *AccountSettingsServiceImpl) SealAccount(ctx context.Context, accountID
 		switch {
 		case writeErr == nil:
 			return &AccountSettingsSeal{Revision: newRev}, nil
-		case kvutil.IsRevisionConflict(writeErr):
+		case kvstore.IsRevisionConflict(writeErr):
 			continue
 		default:
 			slog.ErrorContext(ctx, "Failed to write sealed account settings record", "accountID", accountID, "err", writeErr)

spinifex/domains/ec2/account/service_impl.go

@@ -13,7 +13,7 @@ import (
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -57,7 +57,7 @@ func NewAccountSettingsServiceImplWithNATS(ctx context.Context, cfg *config.Conf
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	settingsKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketAccountSettings, 10)
+	settingsKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketAccountSettings, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create account settings KV bucket: %w", err)
 	}
@@ -115,13 +115,13 @@ func mapSettingsError(ctx context.Context, accountID string, err error) error {
 		return errors.New(awserrors.ErrorBlocked)
 	case errors.Is(err, errSettingsContended):
 		slog.ErrorContext(ctx, "Account settings CAS retries exhausted under contention", "accountID", accountID)
-	case errors.Is(err, kvutil.ErrRead):
+	case errors.Is(err, kvstore.ErrRead):
 		slog.ErrorContext(ctx, "Failed to read account settings from KV", "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrDecode):
+	case errors.Is(err, kvstore.ErrDecode):
 		slog.ErrorContext(ctx, "Corrupt account settings record in KV", "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrEncode):
+	case errors.Is(err, kvstore.ErrEncode):
 		slog.ErrorContext(ctx, "Failed to marshal account settings record", "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrWrite):
+	case errors.Is(err, kvstore.ErrWrite):
 		slog.ErrorContext(ctx, "Failed to write account settings to KV", "accountID", accountID, "err", err)
 	default:
 		slog.ErrorContext(ctx, "Account settings operation failed", "accountID", accountID, "err", err)
@@ -133,7 +133,7 @@ func mapSettingsError(ctx context.Context, accountID string, err error) error {
 // changed is always true, so every success is a leader-accepted write over an
 // active value; a seal landing between the read and the write fails the CAS.
 func (s *AccountSettingsServiceImpl) mutateSettings(ctx context.Context, accountID string, set func(*AccountSettingsRecord)) error {
-	_, err := kvutil.Update(ctx, s.settingsKV, settingsKey(accountID), kvutil.CASConfig{
+	_, err := kvstore.Update(ctx, s.settingsKV, settingsKey(accountID), kvstore.CASConfig{
 		CreateIfAbsent: true,
 		Exhausted:      func(string, int) error { return errSettingsContended },
 	}, func(rec *AccountSettingsRecord) (bool, error) {

spinifex/domains/ec2/ebs/metadata/metadata.go

@@ -110,6 +110,23 @@ type AMI struct {
 	Tags            map[string]string `json:"tags,omitempty"`
 	SourceDigest    *ImageDigest      `json:"source_digest,omitempty"`
 	State           string            `json:"state,omitempty"`
+	// UEFIData names the image's own UEFI variable store object; nil launches
+	// from the node's VARS template.
+	UEFIData *UEFIDataRef `json:"uefi_data,omitempty"`
+}
+
+// UEFIDataRef names an image's UEFI variable store object and the firmware it
+// was written under, so a launch can check the firmware without reading it.
+type UEFIDataRef struct {
+	Key      string       `json:"key"`
+	Firmware UEFIFirmware `json:"firmware"`
+}
+
+// UEFIFirmware is the firmware a variable store was written under.
+type UEFIFirmware struct {
+	Arch       string `json:"arch"`
+	VarsSize   int64  `json:"vars_size"`
+	CodeSHA256 string `json:"code_sha256"`
 }
 
 // ImageDigest identifies the artifact an AMI was imported from, not the

spinifex/domains/ec2/ebs/metadata/metadata_test.go

@@ -73,6 +73,16 @@ func TestAMIWithoutSourceDigestStillDecodes(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "ami-1", got.ImageID)
 	assert.Nil(t, got.SourceDigest)
+	assert.Nil(t, got.UEFIData, "a document with no uefi_data launches from the template")
+}
+
+// The UEFI store reference rides at schema version 2 under its JSON names.
+func TestAMIUEFIDataDecodesAtSchemaVersion2(t *testing.T) {
+	got, err := UnmarshalAMI([]byte(`{"schema_version":2,"image_id":"ami-1","uefi_data":` +
+		`{"key":"uefi/ami-1/k","firmware":{"arch":"x86_64","vars_size":540672,"code_sha256":"abc"}}}`))
+	require.NoError(t, err)
+	require.NotNil(t, got.UEFIData)
+	assert.Equal(t, UEFIDataRef{Key: "uefi/ami-1/k", Firmware: UEFIFirmware{Arch: "x86_64", VarsSize: 540672, CodeSHA256: "abc"}}, *got.UEFIData)
 }
 
 func TestMetadataRejectsUnknownSchemaVersion(t *testing.T) {

spinifex/domains/ec2/eigw/service_impl.go

@@ -19,7 +19,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -56,7 +56,7 @@ func NewEgressOnlyIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	eigwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketEgressOnlyIGW, 10)
+	eigwKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketEgressOnlyIGW, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketEgressOnlyIGW, err)
 	}
@@ -65,7 +65,7 @@ func NewEgressOnlyIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config
 	}
 
 	// Get or create VPC KV bucket for cross-resource ownership validation
-	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
@@ -90,7 +90,7 @@ func (s *EgressOnlyIGWServiceImpl) CreateEgressOnlyInternetGateway(ctx context.C
 		slog.ErrorContext(ctx, "VPC KV unavailable, cannot verify VPC ownership")
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, *input.VpcId)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, *input.VpcId)); err != nil {
 		slog.WarnContext(ctx, "CreateEgressOnlyInternetGateway: VPC not found for account", "vpcId", *input.VpcId, "accountID", accountID)
 		return nil, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", *input.VpcId)
 	}
@@ -109,7 +109,7 @@ func (s *EgressOnlyIGWServiceImpl) CreateEgressOnlyInternetGateway(ctx context.C
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal Egress-only IGW record: %w", err)
 	}
-	if _, err := s.eigwKV.Put(ctx, kvutil.AccountKey(accountID, eigwID), data); err != nil {
+	if _, err := s.eigwKV.Put(ctx, kvstore.AccountKey(accountID, eigwID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -127,7 +127,7 @@ func (s *EgressOnlyIGWServiceImpl) DeleteEgressOnlyInternetGateway(ctx context.C
 	}
 
 	eigwID := *input.EgressOnlyInternetGatewayId
-	key := kvutil.AccountKey(accountID, eigwID)
+	key := kvstore.AccountKey(accountID, eigwID)
 
 	// Verify the EIGW exists before deleting
 	if _, err := s.eigwKV.Get(ctx, key); err != nil {
@@ -178,7 +178,7 @@ func (s *EgressOnlyIGWServiceImpl) DescribeEgressOnlyInternetGateways(ctx contex
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {

spinifex/domains/ec2/eigw/service_impl_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -23,7 +23,7 @@ func setupTestEIGWService(t *testing.T) *EgressOnlyIGWServiceImpl {
 	// Create VPC KV bucket and register test VPCs so fail-closed ownership checks pass
 	vpcEntries := map[string][]byte{}
 	for _, vpcID := range []string{"vpc-test123", "vpc-tagged", "vpc-tagged2"} {
-		vpcEntries[kvutil.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
+		vpcEntries[kvstore.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
 	}
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, vpcEntries)
 
@@ -312,7 +312,7 @@ func TestCreateEgressOnlyInternetGateway_CrossAccountVPCRejected(t *testing.T) {
 	// Create VPC KV bucket with a VPC owned by testAccountID
 	vpcID := "vpc-alpha456"
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, map[string][]byte{
-		kvutil.AccountKey(testAccountID, vpcID): []byte(`{"vpc_id":"vpc-alpha456","state":"available"}`),
+		kvstore.AccountKey(testAccountID, vpcID): []byte(`{"vpc_id":"vpc-alpha456","state":"available"}`),
 	})
 
 	var err error

spinifex/domains/ec2/eip/allocate_idempotency.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -36,7 +36,7 @@ func (s *EIPServiceImpl) allocateOnce(ctx context.Context, accountID string, all
 	if key == "" || s.idemKV == nil {
 		return alloc()
 	}
-	kvKey := kvutil.AccountKey(accountID, idempotencyKeyHash(key))
+	kvKey := kvstore.AccountKey(accountID, idempotencyKeyHash(key))
 
 	if out := s.cachedAllocation(ctx, kvKey); out != nil {
 		slog.WarnContext(ctx, "AllocateAddress: returning first result for a retried request",

spinifex/domains/ec2/eip/lease_rebind.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/domains/network/projection"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/topology"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -80,7 +80,7 @@ func (s *EIPServiceImpl) AllocationExists(ctx context.Context, allocationID stri
 	}
 	suffix := "." + allocationID
 	for _, k := range keys {
-		if k != kvutil.VersionKey && strings.HasSuffix(k, suffix) {
+		if k != kvstore.VersionKey && strings.HasSuffix(k, suffix) {
 			return true, nil
 		}
 	}
@@ -98,7 +98,7 @@ func (s *EIPServiceImpl) findByAllocationIDAnyAccount(ctx context.Context, alloc
 
 	suffix := "." + allocationID
 	for _, k := range keys {
-		if k == kvutil.VersionKey || !strings.HasSuffix(k, suffix) {
+		if k == kvstore.VersionKey || !strings.HasSuffix(k, suffix) {
 			continue
 		}
 		entry, err := s.eipKV.Get(ctx, k)

spinifex/domains/ec2/eip/service_impl.go

@@ -20,7 +20,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -66,7 +66,7 @@ func NewEIPServiceImpl(ctx context.Context, natsConn *nats.Conn, externalIPAM *e
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	eipKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketEIPs, 10)
+	eipKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketEIPs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketEIPs, err)
 	}
@@ -76,7 +76,7 @@ func NewEIPServiceImpl(ctx context.Context, natsConn *nats.Conn, externalIPAM *e
 
 	// TTL'd, so a failure to create it is not worth refusing to serve EIPs over:
 	// a nil bucket only costs post-completion dedupe.
-	idemKV, err := kvutil.GetOrCreateBucketWithTTL(ctx, js, KVBucketEIPIdempotency, 1, eipIdempotencyTTL)
+	idemKV, err := kvstore.GetOrCreateBucketWithTTL(ctx, js, KVBucketEIPIdempotency, 1, eipIdempotencyTTL)
 	if err != nil {
 		slog.Warn("EIP service: idempotency bucket unavailable; a retried AllocateAddress may allocate twice",
 			"bucket", KVBucketEIPIdempotency, "err", err)
@@ -146,7 +146,7 @@ func (s *EIPServiceImpl) allocateAddress(ctx context.Context, input *ec2.Allocat
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal EIP record: %w", err)
 	}
-	if _, err := s.eipKV.Put(ctx, kvutil.AccountKey(accountID, allocID), data); err != nil {
+	if _, err := s.eipKV.Put(ctx, kvstore.AccountKey(accountID, allocID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -167,7 +167,7 @@ func (s *EIPServiceImpl) ReleaseAddress(ctx context.Context, input *ec2.ReleaseA
 	}
 
 	allocID := *input.AllocationId
-	key := kvutil.AccountKey(accountID, allocID)
+	key := kvstore.AccountKey(accountID, allocID)
 
 	entry, err := s.eipKV.Get(ctx, key)
 	if err != nil {
@@ -206,7 +206,7 @@ func (s *EIPServiceImpl) AssociateAddress(ctx context.Context, input *ec2.Associ
 	}
 
 	allocID := *input.AllocationId
-	key := kvutil.AccountKey(accountID, allocID)
+	key := kvstore.AccountKey(accountID, allocID)
 
 	entry, err := s.eipKV.Get(ctx, key)
 	if err != nil {
@@ -536,7 +536,7 @@ func (s *EIPServiceImpl) findByENI(ctx context.Context, accountID, eniID string)
 		if err := ctx.Err(); err != nil {
 			return nil, err
 		}
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.eipKV.Get(ctx, k)
@@ -590,7 +590,7 @@ func (s *EIPServiceImpl) DescribeAddresses(ctx context.Context, input *ec2.Descr
 
 	var addresses []*ec2.Address
 	for _, k := range keys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -699,7 +699,7 @@ func (s *EIPServiceImpl) DescribeAddressesAttribute(ctx context.Context, input *
 			if id == nil {
 				continue
 			}
-			key := kvutil.AccountKey(accountID, *id)
+			key := kvstore.AccountKey(accountID, *id)
 			entry, err := s.eipKV.Get(ctx, key)
 			if err != nil {
 				continue // not found, skip
@@ -722,7 +722,7 @@ func (s *EIPServiceImpl) DescribeAddressesAttribute(ctx context.Context, input *
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
 		for _, k := range keys {
-			if k == kvutil.VersionKey {
+			if k == kvstore.VersionKey {
 				continue
 			}
 			if !strings.HasPrefix(k, prefix) {
@@ -819,7 +819,7 @@ func (s *EIPServiceImpl) findByAssociationID(ctx context.Context, accountID, ass
 	}
 
 	for _, k := range keys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -857,7 +857,7 @@ func (s *EIPServiceImpl) AssociatedPublicIPForInstance(ctx context.Context, acco
 		return "", false
 	}
 	for _, k := range keys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.eipKV.Get(ctx, k)
@@ -875,14 +875,14 @@ func (s *EIPServiceImpl) AssociatedPublicIPForInstance(ctx context.Context, acco
 	return "", false
 }
 
-// ReleaseAddressByInstanceID disassociates and releases every EIP still recorded
-// against instanceID, across all accounts. Idempotent backstop for system-VM
-// teardown paths that lose the cached allocation ID — when the VM is already
-// gone from the manager the per-instance release never runs, so an
-// internet-facing ALB's EIP would otherwise orphan. No-op when nothing matches.
-func (s *EIPServiceImpl) ReleaseAddressByInstanceID(instanceID string) error {
+// ReleaseAllocationOnENI disassociates and releases allocID, wherever its record
+// lives across accounts, but only while it is still associated with eniID. It is
+// the idempotent backstop for the launch allocation of a system VM whose owning
+// node never ran the per-instance release. Any other address, including one a
+// customer associated to the same ENI, is left alone. No-op when nothing matches.
+func (s *EIPServiceImpl) ReleaseAllocationOnENI(allocID, eniID string) error {
 	ctx := context.Background()
-	if instanceID == "" {
+	if allocID == "" || eniID == "" {
 		return nil
 	}
 	keys, err := s.eipKV.Keys(ctx)
@@ -892,9 +892,9 @@ func (s *EIPServiceImpl) ReleaseAddressByInstanceID(instanceID string) error {
 		}
 		return fmt.Errorf("list eip keys: %w", err)
 	}
-	var errs []error
 	for _, k := range keys {
-		if k == kvutil.VersionKey {
+		accountID, keyAlloc, found := strings.Cut(k, ".")
+		if !found || keyAlloc != allocID {
 			continue
 		}
 		entry, err := s.eipKV.Get(ctx, k)
@@ -905,28 +905,23 @@ func (s *EIPServiceImpl) ReleaseAddressByInstanceID(instanceID string) error {
 		if err := json.Unmarshal(entry.Value(), &record); err != nil {
 			continue
 		}
-		if record.InstanceId != instanceID || record.AllocationId == "" {
+		if record.AllocationId != allocID || record.ENIId != eniID || record.AssociationId == "" {
 			continue
 		}
-		// Key is "{accountID}.{allocID}"; the account scopes the release calls.
-		accountID, _, found := strings.Cut(k, ".")
-		if !found {
-			continue
-		}
-		if record.AssociationId != "" {
-			if _, err := s.DisassociateAddress(ctx, &ec2.DisassociateAddressInput{
-				AssociationId: aws.String(record.AssociationId),
-			}, accountID); err != nil {
-				errs = append(errs, fmt.Errorf("disassociate %s: %w", record.AllocationId, err))
-			}
+		var errs []error
+		if _, err := s.DisassociateAddress(ctx, &ec2.DisassociateAddressInput{
+			AssociationId: aws.String(record.AssociationId),
+		}, accountID); err != nil {
+			errs = append(errs, fmt.Errorf("disassociate %s: %w", allocID, err))
 		}
 		if _, err := s.ReleaseAddress(ctx, &ec2.ReleaseAddressInput{
-			AllocationId: aws.String(record.AllocationId),
+			AllocationId: aws.String(allocID),
 		}, accountID); err != nil {
-			errs = append(errs, fmt.Errorf("release %s: %w", record.AllocationId, err))
+			errs = append(errs, fmt.Errorf("release %s: %w", allocID, err))
 		}
+		return errors.Join(errs...)
 	}
-	return errors.Join(errs...)
+	return nil
 }
 
 // publishNATEvent publishes a NAT lifecycle event to NATS for vpcd (fire-and-forget).

spinifex/domains/ec2/eip/service_impl_test.go

@@ -176,29 +176,31 @@ func TestEIP_ReleaseWhileAssociated(t *testing.T) {
 	assert.Equal(t, "Address "+record.PublicIp+" is in use.", message)
 }
 
-func TestEIP_ReleaseByInstanceID_ReclaimsAssociatedEIP(t *testing.T) {
+func TestEIP_ReleaseAllocationOnENI_ReclaimsAssociatedEIP(t *testing.T) {
 	svc, _, _ := setupTestEIP(t)
 
 	out, err := svc.AllocateAddress(context.Background(), &ec2.AllocateAddressInput{}, testAccountID)
 	require.NoError(t, err)
 	allocID := *out.AllocationId
 
-	// Mark the EIP associated with a system instance, as an internet-facing ALB
-	// leaves it. ENIId stays empty so disassociate skips the VPC-service lookup.
+	// Mark the EIP associated with an ENI, as an internet-facing ALB leaves it.
 	entry, err := svc.eipKV.Get(t.Context(), testAccountID+"."+allocID)
 	require.NoError(t, err)
 	var record EIPRecord
 	require.NoError(t, json.Unmarshal(entry.Value(), &record))
 	record.State = "associated"
 	record.AssociationId = "eipassoc-test"
-	record.InstanceId = "i-alb-test"
+	record.ENIId = "eni-alb"
 	data, err := json.Marshal(record)
 	require.NoError(t, err)
 	_, err = svc.eipKV.Update(t.Context(), testAccountID+"."+allocID, data, entry.Revision())
 	require.NoError(t, err)
 
-	// Backstop release by instance disassociates then frees the allocation.
-	require.NoError(t, svc.ReleaseAddressByInstanceID("i-alb-test"))
+	// The backstop disassociates then frees the allocation, but not for another ENI.
+	require.NoError(t, svc.ReleaseAllocationOnENI(allocID, "eni-other"))
+	_, err = svc.eipKV.Get(t.Context(), testAccountID+"."+allocID)
+	require.NoError(t, err, "an allocation on a different ENI is left alone")
+	require.NoError(t, svc.ReleaseAllocationOnENI(allocID, "eni-alb"))
 
 	_, err = svc.ReleaseAddress(context.Background(), &ec2.ReleaseAddressInput{AllocationId: aws.String(allocID)}, testAccountID)
 	assert.Error(t, err)
@@ -322,21 +324,22 @@ func TestEIP_DisassociateByENI_SkipsMalformedRecord(t *testing.T) {
 	assert.True(t, found)
 }
 
-func TestEIP_ReleaseByInstanceID_NoMatchIsNoOp(t *testing.T) {
+func TestEIP_ReleaseAllocationOnENI_NoMatchIsNoOp(t *testing.T) {
 	svc, _, _ := setupTestEIP(t)
 
 	out, err := svc.AllocateAddress(context.Background(), &ec2.AllocateAddressInput{}, testAccountID)
 	require.NoError(t, err)
 	allocID := *out.AllocationId
 
-	// An instance with no recorded EIP must leave existing allocations untouched.
-	require.NoError(t, svc.ReleaseAddressByInstanceID("i-unrelated"))
+	// An unknown allocation, or one not associated to the ENI, leaves others untouched.
+	require.NoError(t, svc.ReleaseAllocationOnENI("eipalloc-unrelated", "eni-x"))
+	require.NoError(t, svc.ReleaseAllocationOnENI(allocID, "eni-x"))
 
 	_, err = svc.eipKV.Get(t.Context(), testAccountID+"."+allocID)
 	assert.NoError(t, err)
 
-	// Empty instance ID is a no-op too.
-	require.NoError(t, svc.ReleaseAddressByInstanceID(""))
+	require.NoError(t, svc.ReleaseAllocationOnENI("", "eni-x"))
+	require.NoError(t, svc.ReleaseAllocationOnENI(allocID, ""))
 }
 
 func TestEIP_ReleaseMissingParams(t *testing.T) {

spinifex/domains/ec2/igw/service_impl.go

@@ -22,7 +22,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -93,7 +93,7 @@ func NewIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	igwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketIGW, 10)
+	igwKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketIGW, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketIGW, err)
 	}
@@ -102,13 +102,13 @@ func NewIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 	}
 
 	// Get or create VPC KV bucket for cross-resource ownership validation
-	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
 
 	// Read EIP associations so detach can refuse while addresses are mapped.
-	eipKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
+	eipKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get EIP KV bucket: %w", err)
 	}
@@ -142,7 +142,7 @@ func (s *IGWServiceImpl) createIGW(ctx context.Context, input *ec2.CreateInterne
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal IGW record: %w", err)
 	}
-	if _, err := s.igwKV.Put(ctx, kvutil.AccountKey(accountID, igwID), data); err != nil {
+	if _, err := s.igwKV.Put(ctx, kvstore.AccountKey(accountID, igwID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -160,7 +160,7 @@ func (s *IGWServiceImpl) DeleteInternetGateway(ctx context.Context, input *ec2.D
 	}
 
 	igwID := *input.InternetGatewayId
-	key := kvutil.AccountKey(accountID, igwID)
+	key := kvstore.AccountKey(accountID, igwID)
 
 	entry, err := s.igwKV.Get(ctx, key)
 	if err != nil {
@@ -236,7 +236,7 @@ func (s *IGWServiceImpl) DescribeInternetGateways(ctx context.Context, input *ec
 	foundIDs := make(map[string]bool)
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -326,7 +326,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A
 
 	igwID := *input.InternetGatewayId
 	vpcID := *input.VpcId
-	key := kvutil.AccountKey(accountID, igwID)
+	key := kvstore.AccountKey(accountID, igwID)
 
 	entry, err := s.igwKV.Get(ctx, key)
 	if err != nil {
@@ -347,7 +347,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A
 		slog.ErrorContext(ctx, "VPC KV unavailable, cannot verify VPC ownership")
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID)); err != nil {
 		slog.WarnContext(ctx, "AttachInternetGateway: VPC not found for account", "vpcId", vpcID, "accountID", accountID)
 		return nil, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
 	}
@@ -388,7 +388,7 @@ func (s *IGWServiceImpl) DetachInternetGateway(ctx context.Context, input *ec2.D
 
 	igwID := *input.InternetGatewayId
 	vpcID := *input.VpcId
-	key := kvutil.AccountKey(accountID, igwID)
+	key := kvstore.AccountKey(accountID, igwID)
 
 	entry, err := s.igwKV.Get(ctx, key)
 	if err != nil {
@@ -449,7 +449,7 @@ func (s *IGWServiceImpl) DetachInternetGateway(ctx context.Context, input *ec2.D
 // associated into vpcID, whether to an interface or a NAT gateway. The listing
 // is filtered server-side to the account's keys.
 func (s *IGWServiceImpl) vpcHasMappedAddress(ctx context.Context, accountID, vpcID string) (bool, error) {
-	lister, err := s.eipKV.ListKeysFiltered(ctx, kvutil.AccountKey(accountID, "*"))
+	lister, err := s.eipKV.ListKeysFiltered(ctx, kvstore.AccountKey(accountID, "*"))
 	if err != nil {
 		return false, err
 	}
@@ -493,11 +493,11 @@ func (s *IGWServiceImpl) CreateAttachedInternetGateway(ctx context.Context, acco
 	if s.vpcKV == nil {
 		return false, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID)); err != nil {
 		return false, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
 	}
 
-	key := kvutil.AccountKey(accountID, igwID)
+	key := kvstore.AccountKey(accountID, igwID)
 	data, err := json.Marshal(IGWRecord{
 		InternetGatewayId: igwID,
 		VpcId:             vpcID,
@@ -558,7 +558,7 @@ func (s *IGWServiceImpl) AttachmentIntent(ctx context.Context, accountID, vpcID
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.igwKV.Get(ctx, key)
@@ -599,7 +599,7 @@ func (s *IGWServiceImpl) AttachmentIntent(ctx context.Context, accountID, vpcID
 // Writing only on the pending transition keeps a pass from waking the drift loop.
 func MarkAttached(ctx context.Context, kv jetstream.KeyValue, recordKey, vpcID string) error {
 	confirmed := false
-	record, err := kvutil.Update(ctx, kv, recordKey, kvutil.CASConfig{}, func(r *IGWRecord) (bool, error) {
+	record, err := kvstore.Update(ctx, kv, recordKey, kvstore.CASConfig{}, func(r *IGWRecord) (bool, error) {
 		// Reset per attempt: a retried CAS may see a record another writer has
 		// already moved out of pending.
 		confirmed = r.VpcId == vpcID && r.AttachState == AttachStatePending

spinifex/domains/ec2/igw/service_impl_test.go

@@ -13,7 +13,7 @@ import (
 	ec2eip "github.com/mulgadc/spinifex/spinifex/domains/ec2/eip"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -55,7 +55,7 @@ func setupTestIGWService(t *testing.T) (*IGWServiceImpl, *nats.Conn) {
 	// Create VPC KV bucket and register test VPCs so fail-closed ownership checks pass
 	vpcEntries := map[string][]byte{}
 	for _, vpcID := range []string{"vpc-test123", "vpc-other", "vpc-lifecycle", "vpc-event-test"} {
-		vpcEntries[kvutil.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
+		vpcEntries[kvstore.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
 	}
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, vpcEntries)
 
@@ -68,7 +68,7 @@ func setupTestIGWService(t *testing.T) (*IGWServiceImpl, *nats.Conn) {
 // gateway come up. Without it an attach stays pending and reports no attachment.
 func confirmAttach(t *testing.T, svc *IGWServiceImpl, igwID, vpcID string) {
 	t.Helper()
-	require.NoError(t, MarkAttached(context.Background(), svc.igwKV, kvutil.AccountKey(testAccountID, igwID), vpcID))
+	require.NoError(t, MarkAttached(context.Background(), svc.igwKV, kvstore.AccountKey(testAccountID, igwID), vpcID))
 }
 
 func createTestIGW(t *testing.T, svc *IGWServiceImpl) string {
@@ -331,7 +331,7 @@ func TestDescribeInternetGateways_LegacyRecordReportsAttachment(t *testing.T) {
 	svc, _ := setupTestIGWService(t)
 	igwID := createTestIGW(t, svc)
 
-	key := kvutil.AccountKey(testAccountID, igwID)
+	key := kvstore.AccountKey(testAccountID, igwID)
 	entry, err := svc.igwKV.Get(context.Background(), key)
 	require.NoError(t, err)
 	var record IGWRecord
@@ -398,7 +398,7 @@ func TestMarkAttached_NoRewriteWhenAlreadyAttached(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 
-	key := kvutil.AccountKey(testAccountID, igwID)
+	key := kvstore.AccountKey(testAccountID, igwID)
 	confirmAttach(t, svc, igwID, "vpc-test123")
 	first, err := svc.igwKV.Get(context.Background(), key)
 	require.NoError(t, err)
@@ -605,7 +605,7 @@ func TestDetachInternetGateway_MappedPublicAddress(t *testing.T) {
 	putEIP := func(rec ec2eip.EIPRecord) {
 		data, err := json.Marshal(rec)
 		require.NoError(t, err)
-		_, err = svc.eipKV.Put(context.Background(), kvutil.AccountKey(testAccountID, rec.AllocationId), data)
+		_, err = svc.eipKV.Put(context.Background(), kvstore.AccountKey(testAccountID, rec.AllocationId), data)
 		require.NoError(t, err)
 	}
 	putEIP(ec2eip.EIPRecord{AllocationId: "eipalloc-other", AssociationId: "eipassoc-other", VpcId: "vpc-other", State: "associated"})
@@ -614,7 +614,7 @@ func TestDetachInternetGateway_MappedPublicAddress(t *testing.T) {
 	// Another account's address naming the same VPC ID is not this account's dependency.
 	foreign, err := json.Marshal(ec2eip.EIPRecord{AllocationId: "eipalloc-foreign", AssociationId: "eipassoc-foreign", VpcId: "vpc-test123", State: "associated"})
 	require.NoError(t, err)
-	_, err = svc.eipKV.Put(context.Background(), kvutil.AccountKey("999999999999", "eipalloc-foreign"), foreign)
+	_, err = svc.eipKV.Put(context.Background(), kvstore.AccountKey("999999999999", "eipalloc-foreign"), foreign)
 	require.NoError(t, err)
 
 	detach := func() error {
@@ -828,7 +828,7 @@ func TestAttachInternetGateway_CrossAccountVPCRejected(t *testing.T) {
 	require.NoError(t, err)
 
 	vpcID := "vpc-alpha123"
-	_, err = vpcKV.Put(t.Context(), kvutil.AccountKey(testAccountID, vpcID), []byte(`{"vpc_id":"vpc-alpha123","state":"available"}`))
+	_, err = vpcKV.Put(t.Context(), kvstore.AccountKey(testAccountID, vpcID), []byte(`{"vpc_id":"vpc-alpha123","state":"available"}`))
 	require.NoError(t, err)
 
 	// Refresh service to pick up the VPC KV bucket

spinifex/domains/ec2/image/service_impl.go

@@ -18,6 +18,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	ec2platform "github.com/mulgadc/spinifex/spinifex/domains/ec2/platform"
 	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -39,6 +40,9 @@ type CreateImageParams struct {
 	RootVolumeID  string
 	SourceImageID string
 	IsRunning     bool // logged only: the provider's CreateSnapshot drains a live volume itself
+	// UEFI is set for an instance that boots through UEFI, whose variable
+	// store the image carries.
+	UEFI bool
 }
 
 // ImageServiceImpl handles AMI image operations with S3 storage.
@@ -49,6 +53,9 @@ type ImageServiceImpl struct {
 	natsConn   *nats.Conn
 	metadata   *ebsmetadata.Store
 	provider   ebsprovider.EBSProvider
+
+	uefiRecords UEFIStoreRecords
+	nodeID      string
 }
 
 // SetEBSProvider injects the provider boundary used by the control plane.
@@ -444,12 +451,23 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		"rootVolumeId", params.RootVolumeID, "amiId", amiID, "snapshotId", snapshotID,
 		"isRunning", params.IsRunning)
 
-	// Step 1: Snapshot root volume
+	// Step 1: Copy the instance's UEFI variable store to the image's own key,
+	// before the snapshot, so a refusal here leaves no snapshot behind.
+	var uefiData *ebsmetadata.UEFIDataRef
+	if params.UEFI {
+		var err error
+		uefiData, err = s.copyInstanceUEFIStore(context.Background(), *input.InstanceId, amiID)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Step 2: Snapshot root volume
 	if err := s.snapshotVolume(params.RootVolumeID, snapshotID, accountID); err != nil {
 		return nil, err
 	}
 
-	// Step 2: Read source volume config for size
+	// Step 3: Read source volume config for size
 	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, params.RootVolumeID)
 	if err != nil {
 		slog.Error("CreateImageFromInstance: failed to read volume metadata", "volumeId", params.RootVolumeID, "err", err)
@@ -457,7 +475,7 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 	}
 	volumeSizeGiB := volMeta.CapacityGiB
 
-	// Step 3: Read source AMI config for architecture, platform, etc.
+	// Step 4: Read source AMI config for architecture, platform, etc.
 	sourceAMI := ebsmetadata.AMI{
 		Architecture:    "x86_64",
 		PlatformDetails: "Linux/UNIX",
@@ -472,13 +490,13 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		sourceAMI = srcCfg
 	}
 
-	// Step 4: Store snapshot metadata
+	// Step 5: Store snapshot metadata
 	if err := s.putSnapshotMetadata(context.Background(), snapshotID, params.RootVolumeID, volumeSizeGiB, accountID); err != nil {
 		slog.Error("CreateImageFromInstance: failed to write snapshot metadata", "snapshotId", snapshotID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// Step 5: Build and store AMI config
+	// Step 6: Build and store AMI config
 	meta := ebsmetadata.AMI{
 		ImageID:         amiID,
 		Name:            name,
@@ -497,10 +515,12 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		Distro:       sourceAMI.Distro,
 		DistroFamily: sourceAMI.DistroFamily,
 		// Snapshot succeeded before this point, so the image is complete.
-		State: "available",
+		State:    "available",
+		UEFIData: uefiData,
 	}
 
 	if err := s.putAMIConfig(context.Background(), amiID, meta); err != nil {
+		// A UEFI object left under the image's prefix is in the orphan report.
 		slog.Error("CreateImageFromInstance: failed to store AMI config", "amiId", amiID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
@@ -856,6 +876,17 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 
 	tags := mergeCopyImageTags(srcMeta.Tags, input.TagSpecifications, aws.BoolValue(input.CopyImageTags))
 
+	uefiData, err := s.copyImageUEFIStore(ctx, srcMeta, newImageID)
+	if err != nil {
+		slog.ErrorContext(ctx, "CopyImage: failed to copy the UEFI variable store",
+			"sourceImageId", sourceImageID, "amiId", newImageID, "orphanSnapshotId", newSnapshotID, "err", err)
+		if delErr := s.metadata.DeleteSnapshot(ctx, accountID, newSnapshotID); delErr != nil {
+			slog.ErrorContext(ctx, "CopyImage: failed to roll back orphaned snapshot metadata",
+				"snapshotId", newSnapshotID, "err", delErr)
+		}
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
 	meta := ebsmetadata.AMI{
 		ImageID:         newImageID,
 		Name:            name,
@@ -876,12 +907,18 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 		SourceDigest:    srcMeta.SourceDigest,
 		// Zero-copy: the new config shares the source's already-durable
 		// snapshot, so the image is complete as soon as this is written.
-		State: "available",
+		State:    "available",
+		UEFIData: uefiData,
 	}
 
 	if err := s.putAMIConfig(ctx, newImageID, meta); err != nil {
+		var orphanUEFIKey string
+		if uefiData != nil {
+			orphanUEFIKey = uefiData.Key
+		}
+		// The copied UEFI object stays under the new image's prefix; the log names it.
 		slog.ErrorContext(ctx, "CopyImage: failed to write AMI config",
-			"amiId", newImageID, "orphanSnapshotId", newSnapshotID, "err", err)
+			"amiId", newImageID, "orphanSnapshotId", newSnapshotID, "orphanUEFIKey", orphanUEFIKey, "err", err)
 		// Best-effort rollback of the orphaned snapshot metadata.
 		if delErr := s.metadata.DeleteSnapshot(ctx, accountID, newSnapshotID); delErr != nil {
 			slog.ErrorContext(ctx, "CopyImage: failed to roll back orphaned snapshot metadata",
@@ -1125,10 +1162,10 @@ func pickRootSnapshotBDM(mappings []*ec2.BlockDeviceMapping, rootDeviceName *str
 	return nil
 }
 
-// DeregisterImage hard-deletes the AMI document. The pre-provider
-// {imageID}/config.json goes too, or the metadata store's legacy read fallback
-// would resurrect the image. Backing snapshot is untouched, so operators run
-// delete-snapshot separately to reclaim block storage.
+// DeregisterImage hard-deletes the image's UEFI store prefix, then the AMI
+// document, so a retry after a failed delete finishes the job. The legacy
+// {imageID}/config.json goes too, or the metadata store's read fallback would
+// resurrect the image. The backing snapshot is untouched, as on AWS.
 func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.DeregisterImageInput, accountID string) (*ec2.DeregisterImageOutput, error) {
 	if input == nil || input.ImageId == nil || *input.ImageId == "" {
 		return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -1140,6 +1177,12 @@ func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.Dereg
 		return nil, err
 	}
 
+	// Only the image's own prefix: a copy or a source instance has its own.
+	if n, err := uefistore.DeleteObjects(ctx, s.store, s.bucketName, imageID); err != nil {
+		slog.ErrorContext(ctx, "DeregisterImage: failed to delete the UEFI variable store", "imageId", imageID, "deleted", n, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
 	if err := s.metadata.DeleteAMI(ctx, imageID); err != nil && !objectstore.IsNoSuchKeyError(err) {
 		slog.ErrorContext(ctx, "DeregisterImage: failed to delete AMI document", "imageId", imageID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)

spinifex/domains/ec2/image/uefi.go

@@ -0,0 +1,101 @@
+package image
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+// uefiPointerAttempts bounds how often CreateImage re-reads a running
+// instance's pointer after the object it named was pruned behind a newer one.
+const uefiPointerAttempts = 3
+
+// UEFIStoreRecords is the store-record read CreateImage makes of its source
+// instance, answered by the stream leader so it sees the last landed pointer.
+type UEFIStoreRecords interface {
+	LeaderGet(ctx context.Context, instanceID string) (*uefistore.UEFIStoreRecord, uint64, error)
+}
+
+var _ UEFIStoreRecords = (*uefistore.Store)(nil)
+
+// SetUEFIStores wires the store records CreateImage reads a UEFI instance's
+// pointer from, and the node ID it writes image store objects as.
+func (s *ImageServiceImpl) SetUEFIStores(records UEFIStoreRecords, nodeID string) {
+	s.uefiRecords = records
+	s.nodeID = nodeID
+}
+
+// copyInstanceUEFIStore copies the object instanceID's store record points at
+// to a fresh key under imageID and returns its reference, or nil when the
+// instance has no record or no pointer. The error is an AWS error code.
+func (s *ImageServiceImpl) copyInstanceUEFIStore(ctx context.Context, instanceID, imageID string) (*ebsmetadata.UEFIDataRef, error) {
+	if s.uefiRecords == nil || s.nodeID == "" {
+		slog.ErrorContext(ctx, "CreateImage: no UEFI store records are wired on this node", "instanceId", instanceID)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	// Once a live pointer is seen, a record that then vanishes is a terminate
+	// racing this request, not an instance without a store.
+	sawPointer := false
+	for range uefiPointerAttempts {
+		rec, _, err := s.uefiRecords.LeaderGet(ctx, instanceID)
+		switch {
+		case errors.Is(err, kvstore.ErrNotFound) && sawPointer:
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		case errors.Is(err, kvstore.ErrNotFound):
+			return nil, nil
+		case err != nil:
+			slog.ErrorContext(ctx, "CreateImage: failed to read the UEFI store record", "instanceId", instanceID, "err", err)
+			return nil, errors.New(awserrors.ErrorServerInternal)
+		case rec.State != uefistore.StateLive:
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		case rec.Pointer == nil:
+			return nil, nil
+		}
+		sawPointer = true
+		ptr := rec.Pointer
+		key, err := uefistore.CopyObject(ctx, s.store, s.bucketName, ptr.Key, uefistore.ObjectMeta{
+			OwnerID: imageID, WriterNode: s.nodeID, Firmware: ptr.Firmware, Source: uefistore.SourceInstance(instanceID),
+		})
+		if errors.Is(err, uefistore.ErrObjectNotFound) {
+			// A write-back moved the pointer and a prune removed this one.
+			continue
+		}
+		if err != nil {
+			slog.ErrorContext(ctx, "CreateImage: failed to copy the UEFI variable store",
+				"instanceId", instanceID, "imageId", imageID, "key", ptr.Key, "err", err)
+			return nil, errors.New(awserrors.ErrorServerInternal)
+		}
+		slog.InfoContext(ctx, "CreateImage: copied the UEFI variable store",
+			"instanceId", instanceID, "imageId", imageID, "sourceKey", ptr.Key, "key", key)
+		return uefistore.AMIRef(key, ptr.Firmware), nil
+	}
+	slog.ErrorContext(ctx, "CreateImage: the UEFI store pointer kept naming a missing object",
+		"instanceId", instanceID, "attempts", uefiPointerAttempts)
+	return nil, errors.New(awserrors.ErrorServerInternal)
+}
+
+// copyImageUEFIStore copies the store src names to a fresh key under imageID,
+// so deregistering either image leaves the other's intact. It returns nil
+// when src names none.
+func (s *ImageServiceImpl) copyImageUEFIStore(ctx context.Context, src ebsmetadata.AMI, imageID string) (*ebsmetadata.UEFIDataRef, error) {
+	store, err := uefistore.ImageVarStoreFromAMI(src)
+	if err != nil || store == nil {
+		return nil, err
+	}
+	if s.nodeID == "" {
+		return nil, errors.New("no node ID is wired to write UEFI store objects as")
+	}
+	key, err := uefistore.CopyObject(ctx, s.store, s.bucketName, store.Key, uefistore.ObjectMeta{
+		OwnerID: imageID, WriterNode: s.nodeID, Firmware: store.Firmware, Source: uefistore.SourceImage(src.ImageID),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("copy UEFI store %s to %s: %w", store.Key, imageID, err)
+	}
+	return uefistore.AMIRef(key, store.Firmware), nil
+}

spinifex/domains/ec2/image/uefi_test.go

@@ -0,0 +1,417 @@
+package image
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awss3 "github.com/aws/aws-sdk-go/service/s3"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	uefiTestInstance = "i-0123456789abcdef0"
+	uefiTestVarsSize = 4096
+)
+
+var uefiTestFirmware = uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: uefiTestVarsSize, CodeSHA256: "code-sha"}
+
+func uefiRegion(tag string) []byte {
+	b := make([]byte, uefiTestVarsSize)
+	copy(b, tag)
+	return b
+}
+
+// uefiImageFixture is an image service wired to a real store-record bucket,
+// with a UEFI instance whose root volume CreateImage can snapshot.
+type uefiImageFixture struct {
+	svc       *ImageServiceImpl
+	snapshots *countingSnapshots
+	objects   *objectstore.MemoryObjectStore
+	records   *uefistore.Store
+	volume    ebsmetadata.Volume
+}
+
+func newUEFIImageFixture(t *testing.T) *uefiImageFixture {
+	t.Helper()
+	svc, objects, provider := setupProviderSnapshotImageService(t)
+	_, _, js := testutil.StartTestJetStream(t)
+	records := uefistore.NewStore(js)
+	require.NoError(t, records.Open(t.Context()))
+	svc.SetUEFIStores(records, "node-a")
+	volume := seedProviderVolume(t, svc, provider, "vol-uefi001", 8)
+	snapshots := &countingSnapshots{EBSProvider: provider}
+	svc.SetEBSProvider(snapshots)
+	return &uefiImageFixture{svc: svc, snapshots: snapshots, objects: objects, records: records, volume: volume}
+}
+
+// countingSnapshots counts the snapshots CreateImage asks the provider for.
+type countingSnapshots struct {
+	ebsprovider.EBSProvider
+
+	created int
+}
+
+func (c *countingSnapshots) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	c.created++
+	return c.EBSProvider.CreateSnapshot(ctx, req)
+}
+
+// assertNoSnapshot checks a refused CreateImage took no snapshot and wrote no
+// snapshot document.
+func (f *uefiImageFixture) assertNoSnapshot(t *testing.T) {
+	t.Helper()
+	assert.Zero(t, f.snapshots.created, "a refused CreateImage takes no snapshot")
+	snaps, err := f.svc.MetadataStore().ListAllSnapshots(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, snaps, "a refused CreateImage writes no snapshot document")
+}
+
+// putInstanceStore puts vars at a fresh key under the instance and points a
+// live store record at it.
+func (f *uefiImageFixture) putInstanceStore(t *testing.T, vars []byte) string {
+	t.Helper()
+	key := f.putObject(t, uefiTestInstance, vars)
+	_, err := f.records.Create(t.Context(), &uefistore.UEFIStoreRecord{
+		InstanceID: uefiTestInstance, AccountID: testAccountID, State: uefistore.StateLive, Generation: 1,
+		Pointer: &uefistore.UEFIVarStoreRef{Key: key, Firmware: uefiTestFirmware, WriterNode: "node-a", WrittenAt: time.Now().UTC()},
+	})
+	require.NoError(t, err)
+	return key
+}
+
+func (f *uefiImageFixture) putObject(t *testing.T, owner string, vars []byte) string {
+	t.Helper()
+	key, err := uefistore.NewObjectKey(owner)
+	require.NoError(t, err)
+	require.NoError(t, uefistore.PutObject(t.Context(), f.objects, testBucket, key, vars, uefistore.ObjectMeta{
+		OwnerID: owner, WriterNode: "node-a", Firmware: uefiTestFirmware, Source: uefistore.SourceWriteBack,
+	}))
+	return key
+}
+
+func (f *uefiImageFixture) createImage(t *testing.T, uefi bool) (string, error) {
+	t.Helper()
+	out, err := f.svc.CreateImageFromInstance(CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String(uefiTestInstance), Name: aws.String("uefi-image")},
+		RootVolumeID: f.volume.VolumeID,
+		UEFI:         uefi,
+	}, testAccountID)
+	if err != nil {
+		return "", err
+	}
+	return aws.StringValue(out.ImageId), nil
+}
+
+func (f *uefiImageFixture) ami(t *testing.T, imageID string) ebsmetadata.AMI {
+	t.Helper()
+	ami, err := f.svc.MetadataStore().GetAMI(t.Context(), imageID)
+	require.NoError(t, err)
+	return ami
+}
+
+func (f *uefiImageFixture) region(t *testing.T, key string) []byte {
+	t.Helper()
+	out, err := f.objects.GetObject(t.Context(), &awss3.GetObjectInput{Bucket: aws.String(testBucket), Key: aws.String(key)})
+	require.NoError(t, err)
+	defer out.Body.Close()
+	zr, err := gzip.NewReader(out.Body)
+	require.NoError(t, err)
+	b, err := io.ReadAll(zr)
+	require.NoError(t, err)
+	return b
+}
+
+func (f *uefiImageFixture) keys(t *testing.T, owner string) []string {
+	t.Helper()
+	keys, err := uefistore.ListObjects(t.Context(), f.objects, testBucket, owner)
+	require.NoError(t, err)
+	return keys
+}
+
+func (f *uefiImageFixture) copyImage(t *testing.T, src, name string) string {
+	t.Helper()
+	out, err := f.svc.CopyImage(t.Context(), validCopyImageServiceInput(src, name), testAccountID)
+	require.NoError(t, err)
+	return aws.StringValue(out.ImageId)
+}
+
+func (f *uefiImageFixture) deregister(t *testing.T, imageID string) error {
+	t.Helper()
+	_, err := f.svc.DeregisterImage(t.Context(), &ec2.DeregisterImageInput{ImageId: aws.String(imageID)}, testAccountID)
+	return err
+}
+
+// CreateImage copies the object the instance's record points at to the
+// image's own prefix, and the AMI document names it with its firmware.
+func TestCreateImage_CopiesTheInstancesUEFIStore(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	instanceKey := f.putInstanceStore(t, uefiRegion("guest variables"))
+
+	imageID, err := f.createImage(t, true)
+	require.NoError(t, err)
+
+	ami := f.ami(t, imageID)
+	require.NotNil(t, ami.UEFIData)
+	assert.Equal(t, []string{ami.UEFIData.Key}, f.keys(t, imageID), "the image has one object under its own prefix")
+	assert.Equal(t, uefiRegion("guest variables"), f.region(t, ami.UEFIData.Key))
+	assert.Equal(t, uefistore.AMIRef(ami.UEFIData.Key, uefiTestFirmware), ami.UEFIData)
+	assert.Equal(t, []string{instanceKey}, f.keys(t, uefiTestInstance), "the instance's store is left as it was")
+
+	store, err := uefistore.ImageVarStoreFromAMI(ami)
+	require.NoError(t, err)
+	assert.Equal(t, &uefistore.ImageVarStore{ImageID: imageID, Key: ami.UEFIData.Key, Firmware: uefiTestFirmware}, store,
+		"the document decodes into the store a launch seeds from")
+}
+
+// A UEFI instance with no store record or no pointer, and a BIOS instance,
+// give an image that launches from the template.
+func TestCreateImage_AnInstanceWithoutAPointerContributesNothing(t *testing.T) {
+	t.Run("no record", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		imageID, err := f.createImage(t, true)
+		require.NoError(t, err)
+		assert.Nil(t, f.ami(t, imageID).UEFIData)
+		assert.Empty(t, f.keys(t, imageID))
+	})
+	t.Run("no pointer", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		_, err := f.records.Create(t.Context(), &uefistore.UEFIStoreRecord{
+			InstanceID: uefiTestInstance, AccountID: testAccountID, State: uefistore.StateLive, Generation: 1,
+		})
+		require.NoError(t, err)
+		imageID, err := f.createImage(t, true)
+		require.NoError(t, err)
+		assert.Nil(t, f.ami(t, imageID).UEFIData)
+	})
+	t.Run("BIOS instance", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		f.putInstanceStore(t, uefiRegion("never read"))
+		f.svc.SetUEFIStores(nil, "")
+		imageID, err := f.createImage(t, false)
+		require.NoError(t, err)
+		assert.Nil(t, f.ami(t, imageID).UEFIData)
+	})
+}
+
+// A UEFI instance's image is refused rather than made without its store when
+// this node cannot read store records, and a terminating store is refused.
+func TestCreateImage_RefusesWhenTheStoreCannotBeCarried(t *testing.T) {
+	t.Run("no records wired", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		f.svc.SetUEFIStores(nil, "")
+		_, err := f.createImage(t, true)
+		require.EqualError(t, err, awserrors.ErrorServerInternal)
+		f.assertNoSnapshot(t)
+	})
+	t.Run("deleting", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		f.putInstanceStore(t, uefiRegion("terminating"))
+		_, err := f.records.MarkDeleting(t.Context(), uefiTestInstance)
+		require.NoError(t, err)
+		_, err = f.createImage(t, true)
+		require.EqualError(t, err, awserrors.ErrorIncorrectInstanceState)
+		f.assertNoSnapshot(t)
+	})
+	t.Run("record unreadable", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		f.svc.SetUEFIStores(failingRecords{}, "node-a")
+		_, err := f.createImage(t, true)
+		require.EqualError(t, err, awserrors.ErrorServerInternal)
+		f.assertNoSnapshot(t)
+	})
+	t.Run("object unreadable", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		key := f.putInstanceStore(t, uefiRegion("x"))
+		_, err := f.objects.PutObject(t.Context(), &awss3.PutObjectInput{
+			Bucket: aws.String(testBucket), Key: aws.String(key), Body: bytes.NewReader([]byte("not gzip")),
+		})
+		require.NoError(t, err)
+		_, err = f.createImage(t, true)
+		require.EqualError(t, err, awserrors.ErrorServerInternal)
+		f.assertNoSnapshot(t)
+	})
+}
+
+// failingRecords cannot read any store record.
+type failingRecords struct{}
+
+func (failingRecords) LeaderGet(context.Context, string) (*uefistore.UEFIStoreRecord, uint64, error) {
+	return nil, 0, assert.AnError
+}
+
+// vanishingRecord shows a live pointer whose object is gone, then no record,
+// as a terminate's reaper leaves it.
+type vanishingRecord struct {
+	stale *uefistore.UEFIStoreRecord
+	reads int
+}
+
+func (v *vanishingRecord) LeaderGet(context.Context, string) (*uefistore.UEFIStoreRecord, uint64, error) {
+	v.reads++
+	if v.reads == 1 {
+		return v.stale, 1, nil
+	}
+	return nil, 0, kvstore.ErrNotFound
+}
+
+// A record that vanishes after CreateImage saw its live pointer is a terminate
+// racing the request, so the image is refused rather than made without its store.
+func TestCreateImage_StoreDeletedMidRequestIsIncorrectInstanceState(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	f.putInstanceStore(t, uefiRegion("x"))
+	stale, _, err := f.records.Get(t.Context(), uefiTestInstance)
+	require.NoError(t, err)
+	stale.Pointer.Key = uefistore.KeyPrefix(uefiTestInstance) + "reaped"
+	f.svc.SetUEFIStores(&vanishingRecord{stale: stale}, "node-a")
+
+	_, err = f.createImage(t, true)
+
+	require.EqualError(t, err, awserrors.ErrorIncorrectInstanceState)
+	f.assertNoSnapshot(t)
+	amis, err := f.svc.MetadataStore().ListAMIs(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, amis)
+}
+
+// laggingPointer answers the first read with a pointer whose object a prune
+// has removed, and every later read with the record as stored.
+type laggingPointer struct {
+	*uefistore.Store
+
+	stale *uefistore.UEFIStoreRecord
+	reads int
+}
+
+func (l *laggingPointer) LeaderGet(ctx context.Context, id string) (*uefistore.UEFIStoreRecord, uint64, error) {
+	l.reads++
+	if l.reads == 1 {
+		return l.stale, 1, nil
+	}
+	return l.Store.LeaderGet(ctx, id)
+}
+
+// A running instance's write-back can move the pointer and prune the old
+// object between CreateImage's read and its get, so the pointer is read again.
+func TestCreateImage_RereadsAPointerWhoseObjectWasPruned(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	f.putInstanceStore(t, uefiRegion("newest"))
+	stale, _, err := f.records.Get(t.Context(), uefiTestInstance)
+	require.NoError(t, err)
+	stale.Pointer.Key = uefistore.KeyPrefix(uefiTestInstance) + "pruned"
+	lagging := &laggingPointer{Store: f.records, stale: stale}
+	f.svc.SetUEFIStores(lagging, "node-a")
+
+	imageID, err := f.createImage(t, true)
+	require.NoError(t, err)
+
+	assert.Equal(t, 2, lagging.reads)
+	assert.Equal(t, uefiRegion("newest"), f.region(t, f.ami(t, imageID).UEFIData.Key))
+}
+
+// CopyImage gives the copy its own object holding the source's region.
+func TestCopyImage_GivesTheCopyItsOwnUEFIStore(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	f.putInstanceStore(t, uefiRegion("source variables"))
+	src, err := f.createImage(t, true)
+	require.NoError(t, err)
+
+	cp := f.copyImage(t, src, "uefi-copy")
+
+	srcRef, cpRef := f.ami(t, src).UEFIData, f.ami(t, cp).UEFIData
+	require.NotNil(t, cpRef)
+	assert.Equal(t, []string{cpRef.Key}, f.keys(t, cp))
+	assert.Equal(t, srcRef.Firmware, cpRef.Firmware)
+	assert.Equal(t, uefiRegion("source variables"), f.region(t, cpRef.Key))
+	assert.Equal(t, []string{srcRef.Key}, f.keys(t, src), "the source keeps its own object")
+}
+
+// A source AMI with no store gives a copy with none.
+func TestCopyImage_SourceWithoutAUEFIStoreCopiesNone(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	seedCopyableAMI(t, f.objects, "ami-src001", "source-ami", testAccountID, "snap-src001", "vol-src001", 8)
+
+	cp := f.copyImage(t, "ami-src001", "copy")
+
+	assert.Nil(t, f.ami(t, cp).UEFIData)
+	assert.Empty(t, f.keys(t, cp))
+}
+
+// A source store that cannot be copied fails the copy and leaves no AMI.
+func TestCopyImage_UnreadableSourceStoreFails(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	seedCopyableAMI(t, f.objects, "ami-src001", "source-ami", testAccountID, "snap-src001", "vol-src001", 8)
+	src := f.ami(t, "ami-src001")
+	src.UEFIData = uefistore.AMIRef(uefistore.KeyPrefix("ami-src001")+"gone", uefiTestFirmware)
+	require.NoError(t, f.svc.MetadataStore().PutAMI(t.Context(), src))
+
+	_, err := f.svc.CopyImage(t.Context(), validCopyImageServiceInput("ami-src001", "copy"), testAccountID)
+
+	require.EqualError(t, err, awserrors.ErrorServerInternal)
+	amis, err := f.svc.MetadataStore().ListAMIs(t.Context())
+	require.NoError(t, err)
+	assert.Len(t, amis, 1, "no copy is registered without its store")
+}
+
+// TestS7_DeregisteringASourceImageLeavesItsCopysStore enforces ADR-0003:S7
+// for DeregisterImage, which account termination also calls: it deletes only
+// the deregistered image's prefix, and the copy's store is its own.
+func TestS7_DeregisteringASourceImageLeavesItsCopysStore(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	instanceKey := f.putInstanceStore(t, uefiRegion("source variables"))
+	src, err := f.createImage(t, true)
+	require.NoError(t, err)
+	cp := f.copyImage(t, src, "uefi-copy")
+
+	require.NoError(t, f.deregister(t, src))
+
+	assert.Empty(t, f.keys(t, src), "the deregistered image's store is gone")
+	assert.Equal(t, uefiRegion("source variables"), f.region(t, f.ami(t, cp).UEFIData.Key),
+		"ADR-0003:S7: deregistering the source must not delete the store its copy boots from")
+	assert.Equal(t, []string{instanceKey}, f.keys(t, uefiTestInstance), "the instance's store is left as it was")
+}
+
+// failingUEFIDelete fails every delete under the UEFI root.
+type failingUEFIDelete struct{ *objectstore.MemoryObjectStore }
+
+func (s failingUEFIDelete) DeleteObject(ctx context.Context, in *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) {
+	if strings.HasPrefix(aws.StringValue(in.Key), uefistore.ObjectRoot) {
+		return nil, assert.AnError
+	}
+	return s.MemoryObjectStore.DeleteObject(ctx, in)
+}
+
+// DeregisterImage deletes the store before the document, so a failed delete
+// keeps the image for a retry to finish.
+func TestDeregisterImage_FailedUEFIStoreDeleteKeepsTheImage(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	f.putInstanceStore(t, uefiRegion("x"))
+	imageID, err := f.createImage(t, true)
+	require.NoError(t, err)
+	f.svc.store = failingUEFIDelete{f.objects}
+
+	require.EqualError(t, f.deregister(t, imageID), awserrors.ErrorServerInternal)
+	assert.NotNil(t, f.ami(t, imageID).UEFIData, "the document stays for the retry")
+
+	f.svc.store = f.objects
+	require.NoError(t, f.deregister(t, imageID))
+	assert.Empty(t, f.keys(t, imageID))
+	_, err = f.svc.MetadataStore().GetAMI(t.Context(), imageID)
+	assert.True(t, objectstore.IsNoSuchKeyError(err), "the retry removes the document: %v", err)
+}
+
+var _ UEFIStoreRecords = (*laggingPointer)(nil)

spinifex/domains/ec2/instance/provider_efi_volume_test.go

@@ -1,126 +0,0 @@
-package instance
-
-import (
-	"bytes"
-	"context"
-	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// varsTemplateBytes stands in for a firmware VARS template. The real one is
-// 540,672 bytes, but nothing here depends on the size beyond it being exact.
-func varsTemplateBytes() []byte { return bytes.Repeat([]byte{0xAB}, 4096) }
-
-func providerEFIService(t *testing.T, provider ebsprovider.EBSProvider) *InstanceServiceImpl {
-	t.Helper()
-	return providerRootVolumeService(t, provider, rootVolumeAMILoader(), objectstore.NewMemoryObjectStore())
-}
-
-// TestCreateEFIVolume_Provider_SeedsFromTemplate is the core of the EFI
-// decouple: the variable store is allocated through the provider with the
-// template as seed bytes, so no control-plane engine is built for it.
-func TestCreateEFIVolume_Provider_SeedsFromTemplate(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	template := varsTemplateBytes()
-
-	require.NoError(t, svc.createEFIVolumeViaProvider(context.Background(), "vol-efiseed001-efi", template))
-
-	volume, err := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{
-		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-efiseed001-efi",
-	})
-	require.NoError(t, err)
-	assert.Equal(t, int64(len(template)), volume.CapacityBytes,
-		"the store must be sized byte-exactly to the VARS template")
-
-	seed, ok := provider.SeedData("vol-efiseed001-efi")
-	require.True(t, ok)
-	assert.Equal(t, template, seed, "the template must reach the provider as seed bytes")
-}
-
-// A provider that rounds capacity to a GiB boundary produces a VARS region
-// pflash refuses, which would surface as an unbootable guest rather than a
-// failed launch. The mismatch has to fail the launch instead.
-func TestCreateEFIVolume_Provider_CapacityMismatchFails(t *testing.T) {
-	provider := &roundingProvider{
-		EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true}),
-	}
-	svc := providerEFIService(t, provider)
-
-	err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efiround01-efi", varsTemplateBytes())
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// A relaunch reissues the same create. Reseeding would overwrite the BootOrder
-// the guest wrote on its first boot, so the second call must be a no-op.
-func TestCreateEFIVolume_Provider_RepeatDoesNotReseed(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	ctx := context.Background()
-	template := varsTemplateBytes()
-
-	require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template))
-	require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template))
-
-	seed, ok := provider.SeedData("vol-efirepeat1-efi")
-	require.True(t, ok)
-	assert.Equal(t, template, seed)
-}
-
-// A seed above the wire bound must fail the launch with an error naming the
-// limit, not reach NATS and fail as an oversized publish.
-func TestCreateEFIVolume_Provider_OversizedTemplateFails(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-
-	err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efitoobig1-efi", make([]byte, ebsprovider.MaxSeedBytes+1))
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// TestPrepareEFIVolume_Provider_RegistersEBSRequest covers the whole function
-// against this host's real firmware, including the name the VM launches with.
-func TestPrepareEFIVolume_Provider_RegistersEBSRequest(t *testing.T) {
-	if _, _, _, err := vm.FirmwarePaths("x86_64"); err != nil {
-		t.Skipf("no UEFI firmware on this host: %v", err)
-	}
-
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	instance := &vm.VM{}
-
-	require.NoError(t, svc.prepareEFIVolume(context.Background(), "vol-efiprepare001", instance, "x86_64"))
-
-	instance.EBSRequests.Mu.Lock()
-	defer instance.EBSRequests.Mu.Unlock()
-	require.Len(t, instance.EBSRequests.Requests, 1)
-	assert.Equal(t, "vol-efiprepare001-efi", instance.EBSRequests.Requests[0].Name)
-	assert.True(t, instance.EBSRequests.Requests[0].EFI)
-	assert.False(t, instance.EBSRequests.Requests[0].Boot)
-
-	seed, ok := provider.SeedData("vol-efiprepare001-efi")
-	require.True(t, ok)
-	assert.NotEmpty(t, seed, "the store must be seeded from this host's VARS template")
-}
-
-// roundingProvider reports a GiB-rounded capacity the way a provider with no
-// sub-GiB volume granularity would.
-type roundingProvider struct {
-	ebsprovider.EBSProvider
-}
-
-func (p *roundingProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) {
-	volume, err := p.EBSProvider.CreateVolume(ctx, req)
-	if err != nil {
-		return nil, err
-	}
-	volume.CapacityBytes = 1 << 30
-	return volume, nil
-}

spinifex/domains/ec2/instance/service_impl.go

@@ -10,7 +10,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
 	"log/slog"
 	"maps"
-	"os"
 	"slices"
 	"sort"
 	"strconv"
@@ -27,6 +26,7 @@ import (
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	ebspolicy "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/policy"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instancetypes"
 	ec2platform "github.com/mulgadc/spinifex/spinifex/domains/ec2/platform"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
@@ -1254,6 +1254,10 @@ func (s *InstanceServiceImpl) StartInstance(ctx context.Context, instance *vm.VM
 				slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
 				return refusal
 			}
+			if errors.Is(err, vm.ErrUEFIClaimHeld) {
+				slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
+				return errInsufficientCapacity
+			}
 			slog.ErrorContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
 			return errors.New(awserrors.ErrorServerInternal)
 		}
@@ -1601,18 +1605,20 @@ func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.Ru
 		return nil, err
 	}
 
-	// Step 2: Create EFI variable store (only when the AMI is UEFI; BIOS
-	// guests must not allocate an orphan VARS volume).
-	if instance.BootMode == "uefi" || instance.BootMode == "uefi-preferred" {
-		arch := instanceArchitecture(s.instanceTypes[*input.InstanceType])
-		err = s.prepareEFIVolume(ctx, imageId, instance, arch)
-		if err != nil {
-			return nil, err
-		}
+	// Step 2: Seed the UEFI variable store, which is not a volume. A failure
+	// terminates the instance, and terminate deletes whatever store it left.
+	image, err := s.imageUEFIVarStore(ctx, instance, aws.StringValue(input.ImageId))
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to read the image's UEFI variable store", "instanceId", instance.ID,
+			"imageId", aws.StringValue(input.ImageId), "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if err := s.vmMgr.SeedUEFIVarStore(ctx, instance, image); err != nil {
+		slog.ErrorContext(ctx, "Failed to seed the UEFI variable store", "instanceId", instance.ID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// The EFI store is internal, so it is not reported; every volume the
-	// request named is.
+	// Every volume the request named is reported.
 	volumeInfos := []VolumeInfo{
 		{
 			VolumeId:            imageId,
@@ -1632,6 +1638,23 @@ func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.Ru
 	return volumeInfos, nil
 }
 
+// imageUEFIVarStore returns the store the launching AMI names for a UEFI
+// instance to seed from, or nil when the instance boots without UEFI or the
+// AMI names none.
+func (s *InstanceServiceImpl) imageUEFIVarStore(ctx context.Context, instance *vm.VM, imageID string) (*uefistore.ImageVarStore, error) {
+	if !vm.UsesUEFIVarStore(instance) || !strings.HasPrefix(imageID, "ami-") {
+		return nil, nil
+	}
+	if s.amiLoader == nil {
+		return nil, errors.New("AMI loader not initialized")
+	}
+	ami, err := s.amiLoader.GetAMIConfig(ctx, imageID)
+	if err != nil {
+		return nil, err
+	}
+	return uefistore.ImageVarStoreFromAMI(ami)
+}
+
 // prepareDataVolumes creates an empty volume for every EBS mapping that is not
 // the root and records each as a non-boot request the launcher attaches. A
 // failure unwinds the volumes this call created, so a refused launch leaves none.
@@ -1874,81 +1897,6 @@ func rootVolumeIOPS(requested int) int {
 	return requested
 }
 
-// prepareEFIVolume creates the per-VM EFI variable store, sized exactly to the
-// firmware VARS template (pflash requires byte-exact size) and seeded from it.
-// arch is "x86_64" | "arm64".
-func (s *InstanceServiceImpl) prepareEFIVolume(ctx context.Context, volumeID string, instance *vm.VM, arch string) error {
-	codePath, varsTemplate, varsSize, err := vm.FirmwarePaths(arch)
-	if err != nil {
-		slog.ErrorContext(ctx, "UEFI firmware not installed on this host", "arch", arch, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	template, err := os.ReadFile(varsTemplate)
-	if err != nil {
-		slog.ErrorContext(ctx, "Failed to read VARS template", "path", varsTemplate, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if int64(len(template)) != varsSize {
-		slog.ErrorContext(ctx, "VARS template size mismatch between stat and read", "path", varsTemplate, "statSize", varsSize, "readSize", len(template))
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	slog.InfoContext(ctx, "Preparing EFI variable store", "arch", arch, "firmwarePath", codePath, "varsTemplate", varsTemplate, "size", varsSize)
-
-	efiVolumeName := fmt.Sprintf("%s-efi", volumeID)
-	if s.ebsProvider == nil {
-		slog.ErrorContext(ctx, "no EBS provider configured", "op", "prepareEFIVolume")
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if err := s.createEFIVolumeViaProvider(ctx, efiVolumeName, template); err != nil {
-		return err
-	}
-
-	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
-		Name: efiVolumeName,
-		Boot: false,
-		EFI:  true,
-	})
-	instance.EBSRequests.Mu.Unlock()
-
-	return nil
-}
-
-// createEFIVolumeViaProvider allocates the EFI variable store through the
-// provider, seeding it with this node's firmware VARS template. The bytes
-// travel with the request so the region matches the firmware that will run.
-func (s *InstanceServiceImpl) createEFIVolumeViaProvider(ctx context.Context, efiVolumeName string, template []byte) error {
-	slog.InfoContext(ctx, "Creating EFI variable store via EBS provider", "name", efiVolumeName, "seedBytes", len(template))
-
-	// The store is internal to the launch, never attachable and filtered out of
-	// DescribeVolumes, so it gets no ebsmetadata document the way a root volume
-	// does. Its lifecycle is the root volume's DeleteVolume.
-	created, err := s.ebsProvider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
-		Versioned:        ebsprovider.NewVersioned(),
-		VolumeID:         efiVolumeName,
-		CapacityRange:    ebsprovider.CapacityRange{RequiredBytes: int64(len(template))},
-		AvailabilityZone: s.config.AZ,
-		SeedData:         template,
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "Provider EFI volume creation failed", "name", efiVolumeName, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if created == nil {
-		slog.ErrorContext(ctx, "Provider returned no volume for the EFI variable store", "name", efiVolumeName)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	// pflash rejects a VARS region that is not byte-exact, so a provider that
-	// rounded the capacity up to a GiB boundary yields a guest that cannot boot.
-	if created.CapacityBytes != int64(len(template)) {
-		slog.ErrorContext(ctx, "Provider EFI volume capacity does not match the VARS template",
-			"name", efiVolumeName, "wantBytes", len(template), "gotBytes", created.CapacityBytes)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	return nil
-}
-
 // checkImageArchitecture refuses an AMI whose architecture the instance type
 // does not support, as AWS does. An unrecorded architecture on either side is
 // not a mismatch.
@@ -2749,7 +2697,7 @@ func (s *InstanceServiceImpl) capacityRefusal(ctx context.Context, instance *vm.
 func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instance *vm.VM, undo func()) error {
 	// Before anything is stamped or claimed, so the record handed back is the
 	// one that was claimed.
-	if err := vm.CheckUEFIVarStore(instance); err != nil {
+	if err := s.vmMgr.CheckUEFIVarStore(ctx, instance); err != nil {
 		slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without exactly one variable store",
 			"instanceId", instance.ID, "err", err)
 		undo()
@@ -2882,6 +2830,10 @@ func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instanc
 		if refusal := volumeHeldElsewhereError(err); refusal != nil {
 			return refusal
 		}
+		// A live node holds the variable store, so another node may take it.
+		if errors.Is(err, vm.ErrUEFIClaimHeld) {
+			return s.capacityRefusal(ctx, instance)
+		}
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 

spinifex/domains/ec2/instance/uefi_seed_test.go

@@ -0,0 +1,171 @@
+package instance
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const testSeedInstance = "i-0a1b2c3d4e5f60718"
+
+type seedInstanceTypes map[string]vm.InstanceTypeSpec
+
+func (r seedInstanceTypes) Resolve(name string) (vm.InstanceTypeSpec, bool) {
+	spec, ok := r[name]
+	return spec, ok
+}
+
+// uefiSeedingService is an instance service whose VM manager seeds stores into
+// a real JetStream's store records and claims and an in-memory object store,
+// under a stand-in x86_64 firmware.
+func uefiSeedingService(t *testing.T) (*InstanceServiceImpl, *ebsprovider.MemoryProvider, *uefistore.Store, *objectstore.MemoryObjectStore) {
+	t.Helper()
+	fwDir := t.TempDir()
+	code := filepath.Join(fwDir, "OVMF_CODE.fd")
+	vars := filepath.Join(fwDir, "OVMF_VARS.fd")
+	require.NoError(t, os.WriteFile(code, []byte("code"), 0o600))
+	require.NoError(t, os.WriteFile(vars, make([]byte, 4096), 0o600))
+	orig, had := vm.FirmwarePathCandidates["x86_64"]
+	vm.FirmwarePathCandidates["x86_64"] = []vm.FirmwareCandidate{{Code: code, VarsTemplate: vars}}
+	t.Cleanup(func() {
+		if had {
+			vm.FirmwarePathCandidates["x86_64"] = orig
+		} else {
+			delete(vm.FirmwarePathCandidates, "x86_64")
+		}
+	})
+
+	_, _, js := testutil.StartTestJetStream(t)
+	records := uefistore.NewStore(js)
+	require.NoError(t, records.Open(t.Context()))
+
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{CrashConsistentSnapshot: true, VolumeEnumeration: true})
+	seedProviderSnapshot(t, provider, "vol-origin", "snap-source")
+	objects := objectstore.NewMemoryObjectStore()
+	svc := providerRootVolumeService(t, provider, rootVolumeAMILoader(), objects)
+	svc.vmMgr = vm.NewManagerWithDeps(vm.Deps{
+		NodeID:        "node-a",
+		InstanceTypes: seedInstanceTypes{"t3.nano": {Architecture: "x86_64", VCPUs: 1, MemoryMiB: 512}},
+		UEFIVarStore: vm.UEFIVarStoreDeps{
+			Objects: objects, Bucket: testRootBucket, DataDir: t.TempDir(), Records: records,
+			Claims: uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a"),
+		},
+	})
+	return svc, provider, records, objects
+}
+
+func uefiLaunch() (*ec2.RunInstancesInput, *vm.VM) {
+	input := &ec2.RunInstancesInput{ImageId: aws.String("ami-1"), InstanceType: aws.String("t3.nano")}
+	return input, &vm.VM{ID: testSeedInstance, AccountID: testRootAccount, InstanceType: "t3.nano", BootMode: "uefi"}
+}
+
+// A UEFI launch seeds its store as an object and a store record, and creates
+// no <root>-efi volume and no EFI request.
+func TestGenerateVolumes_UEFILaunchSeedsAStoreAndCreatesNoEFIVolume(t *testing.T) {
+	svc, provider, records, objects := uefiSeedingService(t)
+	input, instance := uefiLaunch()
+
+	infos, err := svc.GenerateVolumes(t.Context(), input, instance)
+	require.NoError(t, err)
+
+	require.Len(t, infos, 1, "only the root volume is reported")
+	instance.EBSRequests.Mu.Lock()
+	for _, req := range instance.EBSRequests.Requests {
+		assert.False(t, req.EFI, "no EFI request is appended")
+		assert.False(t, strings.HasSuffix(req.Name, "-efi"), req.Name)
+	}
+	instance.EBSRequests.Mu.Unlock()
+	listed, err := provider.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	for _, v := range listed.Volumes {
+		assert.False(t, strings.HasSuffix(v.ID, "-efi"), "no EFI volume is created: %s", v.ID)
+	}
+
+	rec, _, err := records.Get(t.Context(), testSeedInstance)
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.StateLive, rec.State)
+	require.NotNil(t, rec.Pointer)
+	keys, err := uefistore.ListObjects(t.Context(), objects, testRootBucket, testSeedInstance)
+	require.NoError(t, err)
+	assert.Equal(t, []string{rec.Pointer.Key}, keys, "the pointer names the one template object")
+}
+
+// A seed that cannot be written fails the launch as InternalError, and no EFI
+// volume is created in its place.
+func TestGenerateVolumes_FailedSeedIsInternalError(t *testing.T) {
+	svc, provider, _, _ := uefiSeedingService(t)
+	svc.vmMgr = vm.NewManager()
+	input, instance := uefiLaunch()
+
+	_, err := svc.GenerateVolumes(t.Context(), input, instance)
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	listed, err := provider.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	for _, v := range listed.Volumes {
+		assert.False(t, strings.HasSuffix(v.ID, "-efi"), v.ID)
+	}
+}
+
+// putImageStore gives ami-1 a store object holding vars and names it in the
+// AMI document the launch reads.
+func putImageStore(t *testing.T, svc *InstanceServiceImpl, objects *objectstore.MemoryObjectStore, key string, vars []byte) {
+	t.Helper()
+	if strings.HasPrefix(key, uefistore.KeyPrefix("ami-1")) {
+		require.NoError(t, uefistore.PutObject(t.Context(), objects, testRootBucket, key, vars, uefistore.ObjectMeta{
+			OwnerID: "ami-1", WriterNode: "node-b", Source: uefistore.SourceInstance("i-0000000000000000a"),
+			Firmware: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: int64(len(vars)), CodeSHA256: "other"},
+		}))
+	}
+	loader := svc.amiLoader.(*fakeAMILoader)
+	ami := loader.byID["ami-1"]
+	ami.UEFIData = uefistore.AMIRef(key, uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: int64(len(vars)), CodeSHA256: "other"})
+	loader.byID["ami-1"] = ami
+}
+
+// An AMI that names a store seeds the new instance's own object from it.
+func TestGenerateVolumes_UEFILaunchSeedsFromTheAMIsStore(t *testing.T) {
+	svc, _, records, objects := uefiSeedingService(t)
+	vars := make([]byte, 4096)
+	copy(vars, "image variables")
+	putImageStore(t, svc, objects, uefistore.KeyPrefix("ami-1")+"k", vars)
+	input, instance := uefiLaunch()
+
+	_, err := svc.GenerateVolumes(t.Context(), input, instance)
+	require.NoError(t, err)
+
+	rec, _, err := records.Get(t.Context(), testSeedInstance)
+	require.NoError(t, err)
+	require.NotNil(t, rec.Pointer)
+	got, _, err := uefistore.ReadObject(t.Context(), objects, testRootBucket, rec.Pointer.Key, 4096)
+	require.NoError(t, err)
+	assert.Equal(t, vars, got)
+}
+
+// An AMI document whose store reference is outside the image's own prefix is
+// refused, and the launch records no store.
+func TestGenerateVolumes_AMIStoreOutsideItsPrefixIsInternalError(t *testing.T) {
+	svc, _, records, objects := uefiSeedingService(t)
+	putImageStore(t, svc, objects, uefistore.KeyPrefix("i-0000000000000000a")+"k", make([]byte, 4096))
+	input, instance := uefiLaunch()
+
+	_, err := svc.GenerateVolumes(t.Context(), input, instance)
+
+	require.EqualError(t, err, awserrors.ErrorServerInternal)
+	_, _, err = records.Get(t.Context(), testSeedInstance)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}

spinifex/domains/ec2/instance/uefistore/cleanup.go

@@ -10,7 +10,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -26,9 +25,9 @@ func (s *Store) Exists(ctx context.Context, instanceID string) (bool, error) {
 	return s.kv.Exists(ctx, instanceID)
 }
 
-// MarkDeleting records that instanceID's store is to be deleted, and drops any
-// write-back still owed. It reports false with no error when the instance has
-// no store record. Only a terminate or an operator's forget may call it.
+// MarkDeleting records that instanceID's store is to be deleted. It reports
+// false with no error when the instance has no store record. Only a terminate
+// or an operator's forget may call it.
 func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, error) {
 	for range markAttempts {
 		rec, rev, err := s.Get(ctx, instanceID)
@@ -42,7 +41,6 @@ func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, erro
 			return true, nil
 		}
 		rec.State = StateDeleting
-		rec.WriteBack = nil
 		_, err = s.CompareAndSet(ctx, rec, rev)
 		if err == nil {
 			return true, nil
@@ -72,7 +70,7 @@ func (s *Store) InstanceIDs(ctx context.Context) ([]string, error) {
 
 	var ids []string
 	for key := range lister.Keys() {
-		if key != kvutil.VersionKey {
+		if key != kvstore.VersionKey {
 			ids = append(ids, key)
 		}
 	}

spinifex/domains/ec2/instance/uefistore/cleanup_test.go

@@ -37,7 +37,7 @@ func keysUnder(t *testing.T, objects objectstore.ObjectStore, prefix string) []s
 }
 
 // Deletion is final: a writer holding the revision of a deleting record, such
-// as a write-back retry, must not bring the store back to live.
+// as a late pointer move, must not bring the store back to live.
 func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	s, _ := openStore(t)
 	rev, err := s.Create(t.Context(), liveRecord())
@@ -47,8 +47,7 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	rev, err = s.CompareAndSet(t.Context(), deleting, rev)
 	require.NoError(t, err)
 
-	revived := liveRecord()
-	revived.WriteBack = writeBack()
+	revived := pointedRecord()
 	_, err = s.CompareAndSet(t.Context(), revived, rev)
 	require.ErrorIs(t, err, uefistore.ErrDeleting)
 
@@ -58,24 +57,9 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	assert.Equal(t, uefistore.StateDeleting, got.State)
 }
 
-// A write-back owed on a deleting store would put an object behind the delete.
-func TestCompareAndSet_RefusesDeletingWithAWriteBack(t *testing.T) {
+func TestMarkDeleting_MovesLiveToDeletingAndKeepsWhatItNames(t *testing.T) {
 	s, _ := openStore(t)
-	rev, err := s.Create(t.Context(), liveRecord())
-	require.NoError(t, err)
-	rec := liveRecord()
-	rec.State = uefistore.StateDeleting
-	rec.WriteBack = writeBack()
-
-	_, err = s.CompareAndSet(t.Context(), rec, rev)
-	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
-}
-
-func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
-	s, _ := openStore(t)
-	rec := liveRecord()
-	rec.WriteBack = writeBack()
-	_, err := s.Create(t.Context(), rec)
+	_, err := s.Create(t.Context(), pointedRecord())
 	require.NoError(t, err)
 
 	has, err := s.MarkDeleting(t.Context(), instanceID)
@@ -85,8 +69,8 @@ func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
 	got, rev, err := s.Get(t.Context(), instanceID)
 	require.NoError(t, err)
 	assert.Equal(t, uefistore.StateDeleting, got.State)
-	assert.Nil(t, got.WriteBack)
 	assert.Equal(t, "111122223333", got.AccountID)
+	assert.Equal(t, pointer("b"), got.Pointer, "the pointer stays readable until the prefix is gone")
 
 	has, err = s.MarkDeleting(t.Context(), instanceID)
 	require.NoError(t, err)

spinifex/domains/ec2/instance/uefistore/fence.go

@@ -0,0 +1,93 @@
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+)
+
+// ClaimPrefix is the kvsync prefix of every store's claim. An instance's claim
+// is ClaimPrefix+instanceID, and names the node that may hold variable changes
+// no object has yet.
+const ClaimPrefix = "uefi."
+
+// NewClaims returns node's group of store claims over store. The daemon builds
+// it once per process, as kvsync requires of every prefix.
+func NewClaims(store kvsync.KVStore, node string) *kvsync.ClaimGroup {
+	return kvsync.NewClaimGroup(store, ClaimPrefix, node)
+}
+
+// ErrFenced is a stamp or pointer move refused because the record carries a
+// claim generation other than the caller's: a later acquisition has stamped
+// it, so the caller no longer owns the store and must change nothing.
+var ErrFenced = errors.New("uefistore: store record is fenced to another claim generation")
+
+// fenceAttempts bounds the read-then-write loops of Stamp and MovePointer. A
+// lost race is re-read, so a concurrent stamp is seen and refused, not retried.
+const fenceAttempts = 5
+
+// Stamp raises instanceID's record to generation, the claim token the caller
+// has just acquired or adopted, and returns the record it stamped, whose
+// pointer the caller boots from. One above generation is ErrFenced, and one
+// deleting is ErrDeleting. A record already at generation is still written by
+// CAS, because a follower can answer the read with an older pointer.
+func (s *Store) Stamp(ctx context.Context, instanceID string, generation uint64) (*UEFIStoreRecord, error) {
+	if generation == 0 {
+		return nil, fmt.Errorf("uefistore: stamp %s: generation 0 is no claim", instanceID)
+	}
+	for range fenceAttempts {
+		rec, rev, err := s.Get(ctx, instanceID)
+		if err != nil {
+			return nil, err
+		}
+		switch {
+		case rec.State == StateDeleting:
+			return nil, fmt.Errorf("%w: %s", ErrDeleting, instanceID)
+		case rec.Generation > generation:
+			return nil, fmt.Errorf("%w: %s is at generation %d, above the claim's %d", ErrFenced, instanceID, rec.Generation, generation)
+		}
+		rec.Generation = generation
+		_, err = s.CompareAndSet(ctx, rec, rev)
+		if err == nil {
+			return rec, nil
+		}
+		if !errors.Is(err, kvstore.ErrConflict) {
+			return nil, err
+		}
+	}
+	return nil, fmt.Errorf("uefistore: stamp %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
+}
+
+// MovePointer points instanceID's record at ref only while the record is at
+// generation. A record at any other generation is ErrFenced and is left
+// unchanged. The leader answers the read, because a caller drops its copy on
+// ErrFenced or ErrDeleting, and the CAS keeps a lost race from succeeding.
+func (s *Store) MovePointer(ctx context.Context, instanceID string, generation uint64, ref UEFIVarStoreRef) error {
+	if err := ref.validate(instanceID); err != nil {
+		return fmt.Errorf("%w: move %s: %w", ErrInvalidRecord, instanceID, err)
+	}
+	for range fenceAttempts {
+		rec, rev, err := s.LeaderGet(ctx, instanceID)
+		if err != nil {
+			return err
+		}
+		switch {
+		case rec.State == StateDeleting:
+			return fmt.Errorf("%w: %s", ErrDeleting, instanceID)
+		case rec.Generation != generation:
+			return fmt.Errorf("%w: %s is at generation %d, and the writer holds %d", ErrFenced, instanceID, rec.Generation, generation)
+		}
+		rec.Pointer = &ref
+		_, err = s.CompareAndSet(ctx, rec, rev)
+		if err == nil {
+			return nil
+		}
+		if !errors.Is(err, kvstore.ErrConflict) {
+			return err
+		}
+	}
+	return fmt.Errorf("uefistore: move %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
+}

spinifex/domains/ec2/instance/uefistore/fence_test.go

@@ -0,0 +1,213 @@
+package uefistore_test
+
+import (
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func revisionOf(t *testing.T, s *uefistore.Store) uint64 {
+	t.Helper()
+	_, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	return rev
+}
+
+func TestStamp_RaisesTheGenerationAndReturnsThePointerToBootFrom(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+
+	rec, err := s.Stamp(t.Context(), instanceID, 9)
+	require.NoError(t, err)
+	assert.Equal(t, uint64(9), rec.Generation)
+	assert.Equal(t, pointer("b"), rec.Pointer)
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rec, got, "the returned record is the one stored")
+}
+
+// A daemon restart adopts its claim at the same generation. The stamp still
+// writes by CAS, so a pointer read from a lagging follower fails the write and
+// is read again rather than booted from.
+func TestStamp_AtTheRecordsGenerationConfirmsTheReadByCAS(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+	before := revisionOf(t, s)
+
+	rec, err := s.Stamp(t.Context(), instanceID, 7)
+	require.NoError(t, err)
+	assert.Equal(t, pointer("b"), rec.Pointer)
+	assert.Equal(t, uint64(7), rec.Generation)
+	assert.Greater(t, revisionOf(t, s), before, "the read must be confirmed by a revision-checked write")
+}
+
+func TestStamp_RefusesAndChangesNothing(t *testing.T) {
+	tests := []struct {
+		name       string
+		deleting   bool
+		generation uint64
+		want       error
+	}{
+		{name: "a later acquisition stamped it", generation: 6, want: uefistore.ErrFenced},
+		{name: "the store is being deleted", deleting: true, generation: 8, want: uefistore.ErrDeleting},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s, _ := openStore(t)
+			rev, err := s.Create(t.Context(), pointedRecord())
+			require.NoError(t, err)
+			if tt.deleting {
+				_, err = s.MarkDeleting(t.Context(), instanceID)
+				require.NoError(t, err)
+				rev = revisionOf(t, s)
+			}
+
+			_, err = s.Stamp(t.Context(), instanceID, tt.generation)
+			require.ErrorIs(t, err, tt.want)
+			assert.Equal(t, rev, revisionOf(t, s))
+		})
+	}
+}
+
+func TestStamp_MissingRecordIsNotFound(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Stamp(t.Context(), instanceID, 1)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+// The record keeps only the current version, so each move replaces the
+// pointer and names nothing it replaced.
+func TestMovePointer_ReplacesThePointer(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+
+	for _, name := range []string{"c", "d"} {
+		require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer(name)))
+	}
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, pointer("d"), got.Pointer)
+	assert.Equal(t, uint64(7), got.Generation)
+}
+
+// A move to the key the record already names is still a CAS at the revision
+// read, so a fenced writer that read a lagging follower cannot see success.
+func TestMovePointer_ToTheCurrentKeyConfirmsTheReadByCAS(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+	before := revisionOf(t, s)
+
+	require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer("b")))
+	assert.Greater(t, revisionOf(t, s), before)
+}
+
+func TestMovePointer_RefusesAndChangesNothing(t *testing.T) {
+	tests := []struct {
+		name       string
+		deleting   bool
+		generation uint64
+		ref        *uefistore.UEFIVarStoreRef
+		want       error
+	}{
+		{name: "an earlier generation", generation: 6, ref: pointer("c"), want: uefistore.ErrFenced},
+		{name: "a later generation than the stamp", generation: 8, ref: pointer("c"), want: uefistore.ErrFenced},
+		{name: "a deleting store", deleting: true, generation: 7, ref: pointer("c"), want: uefistore.ErrDeleting},
+		{name: "a key under another instance", generation: 7,
+			ref:  &uefistore.UEFIVarStoreRef{Key: uefistore.KeyPrefix("i-0fedcba9876543210") + "c", Firmware: pointer("c").Firmware, WriterNode: "n"},
+			want: uefistore.ErrInvalidRecord},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s, _ := openStore(t)
+			_, err := s.Create(t.Context(), pointedRecord())
+			require.NoError(t, err)
+			if tt.deleting {
+				_, err = s.MarkDeleting(t.Context(), instanceID)
+				require.NoError(t, err)
+			}
+			rev := revisionOf(t, s)
+
+			err = s.MovePointer(t.Context(), instanceID, tt.generation, *tt.ref)
+			require.ErrorIs(t, err, tt.want)
+			got, gotRev, err := s.Get(t.Context(), instanceID)
+			require.NoError(t, err)
+			assert.Equal(t, rev, gotRev)
+			assert.Equal(t, pointer("b"), got.Pointer)
+		})
+	}
+}
+
+// claimedStore is a store record and a claim group per node over one
+// JetStream, as two daemons see them.
+type claimedStore struct {
+	records *uefistore.Store
+	a, b    *kvsync.Claim
+}
+
+func newClaimedStore(t *testing.T) *claimedStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	records := uefistore.NewStore(js)
+	require.NoError(t, records.Open(t.Context()))
+	_, err := records.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	return &claimedStore{
+		records: records,
+		a:       uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a").Get(instanceID),
+		b:       uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-b").Get(instanceID),
+	}
+}
+
+// A node partitioned long enough to be taken over is fenced by the new
+// owner's stamp: its pointer move fails and the record keeps what it had.
+func TestFence_TheOldHoldersMoveFailsAfterTheNewOwnersStamp(t *testing.T) {
+	c := newClaimedStore(t)
+	first, err := c.a.TryLock(t.Context())
+	require.NoError(t, err)
+	_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
+	require.NoError(t, err)
+	require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("b")))
+
+	second, err := c.b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+	_, err = c.records.Stamp(t.Context(), instanceID, second.Generation)
+	require.NoError(t, err)
+	rev := revisionOf(t, c.records)
+
+	err = c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("late"))
+	require.ErrorIs(t, err, uefistore.ErrFenced)
+	got, gotRev, err := c.records.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, pointer("b"), got.Pointer)
+	assert.Equal(t, second.Generation, got.Generation)
+}
+
+// The new owner reads the pointer after it stamps, so a move the old holder
+// landed between the takeover and the stamp is the one it boots from.
+func TestFence_AMoveBetweenTakeoverAndStampIsThePointerTheNewOwnerBoots(t *testing.T) {
+	c := newClaimedStore(t)
+	first, err := c.a.TryLock(t.Context())
+	require.NoError(t, err)
+	_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
+	require.NoError(t, err)
+
+	second, err := c.b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+	require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("landed")))
+
+	rec, err := c.records.Stamp(t.Context(), instanceID, second.Generation)
+	require.NoError(t, err)
+	assert.Equal(t, keyOf("landed"), rec.Pointer.Key)
+}

spinifex/domains/ec2/instance/uefistore/image.go

@@ -0,0 +1,145 @@
+package uefistore
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+const (
+	// GetTimeout bounds one get of a store object, a fixed-size region.
+	GetTimeout = 10 * time.Second
+
+	// bodySlack is how far a compressed body may exceed its region before it
+	// is refused; gzip grows incompressible data by far less.
+	bodySlack = 1 << 20
+)
+
+// ErrObjectNotFound is a store object that does not exist.
+var ErrObjectNotFound = errors.New("uefistore: store object does not exist")
+
+// SourceImage is the source of a store copied from imageID's store.
+func SourceImage(imageID string) string { return "image:" + imageID }
+
+// SourceInstance is the source of an image store copied from instanceID's.
+func SourceInstance(instanceID string) string { return "instance:" + instanceID }
+
+// ImageVarStore is the store an AMI document names: one object under the
+// image's own prefix and the firmware it was written under.
+type ImageVarStore struct {
+	ImageID  string
+	Key      string
+	Firmware FirmwareIdentity
+}
+
+// ImageVarStoreFromAMI returns the store ami names, or nil when it names none.
+// A reference whose key is not one object under the image's own prefix, or
+// that names no firmware, is ErrInvalidObject.
+func ImageVarStoreFromAMI(ami ebsmetadata.AMI) (*ImageVarStore, error) {
+	ref := ami.UEFIData
+	if ref == nil {
+		return nil, nil
+	}
+	if !strings.HasPrefix(ami.ImageID, "ami-") || !ownsKey(ami.ImageID, ref.Key) {
+		return nil, fmt.Errorf("%w: %s names UEFI store %q outside %s", ErrInvalidObject, ami.ImageID, ref.Key, KeyPrefix(ami.ImageID))
+	}
+	if ref.Firmware.Arch == "" || ref.Firmware.VarsSize <= 0 {
+		return nil, fmt.Errorf("%w: %s names UEFI store %q with no firmware arch and VARS size", ErrInvalidObject, ami.ImageID, ref.Key)
+	}
+	return &ImageVarStore{
+		ImageID: ami.ImageID,
+		Key:     ref.Key,
+		Firmware: FirmwareIdentity{
+			Arch: ref.Firmware.Arch, VarsSize: ref.Firmware.VarsSize, CodeSHA256: ref.Firmware.CodeSHA256,
+		},
+	}, nil
+}
+
+// AMIRef returns the reference an AMI document records for key under fw.
+func AMIRef(key string, fw FirmwareIdentity) *ebsmetadata.UEFIDataRef {
+	return &ebsmetadata.UEFIDataRef{
+		Key:      key,
+		Firmware: ebsmetadata.UEFIFirmware{Arch: fw.Arch, VarsSize: fw.VarsSize, CodeSHA256: fw.CodeSHA256},
+	}
+}
+
+// ReadObject gets key within GetTimeout and returns its decompressed region,
+// which must be exactly varsSize bytes, with the object's metadata. A missing
+// object is ErrObjectNotFound and a malformed one ErrInvalidObject.
+func ReadObject(ctx context.Context, objects objectstore.ObjectStore, bucket, key string, varsSize int64) ([]byte, map[string]*string, error) {
+	getCtx, cancel := context.WithTimeout(ctx, GetTimeout)
+	defer cancel()
+
+	out, err := objects.GetObject(getCtx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return nil, nil, fmt.Errorf("%w: %s", ErrObjectNotFound, key)
+		}
+		return nil, nil, fmt.Errorf("get UEFI store %s: %w", key, err)
+	}
+	defer out.Body.Close()
+
+	limit := varsSize + bodySlack
+	raw, err := io.ReadAll(io.LimitReader(out.Body, limit+1))
+	if err != nil {
+		return nil, nil, fmt.Errorf("read UEFI store %s: %w", key, err)
+	}
+	if int64(len(raw)) > limit {
+		return nil, nil, fmt.Errorf("%w: %s is over %d bytes compressed", ErrInvalidObject, key, limit)
+	}
+	gz, err := gzip.NewReader(bytes.NewReader(raw))
+	if err != nil {
+		return nil, nil, fmt.Errorf("%w: %s is not gzip: %w", ErrInvalidObject, key, err)
+	}
+	region, err := io.ReadAll(io.LimitReader(gz, varsSize+1))
+	if err != nil {
+		return nil, nil, fmt.Errorf("%w: %s does not decompress: %w", ErrInvalidObject, key, err)
+	}
+	if int64(len(region)) != varsSize {
+		return nil, nil, fmt.Errorf("%w: %s holds a %d-byte region, not the %d bytes its firmware needs",
+			ErrInvalidObject, key, len(region), varsSize)
+	}
+	return region, out.Metadata, nil
+}
+
+// CopyObject reads srcKey, a region of meta.Firmware's VARS size, and puts it
+// at a fresh key under meta.OwnerID, which it returns. A meta with no CodePath
+// takes the source object's.
+func CopyObject(ctx context.Context, objects objectstore.ObjectStore, bucket, srcKey string, meta ObjectMeta) (string, error) {
+	region, srcMeta, err := ReadObject(ctx, objects, bucket, srcKey, meta.Firmware.VarsSize)
+	if err != nil {
+		return "", err
+	}
+	if meta.CodePath == "" {
+		meta.CodePath = metadataValue(srcMeta, "firmware-code-path")
+	}
+	key, err := NewObjectKey(meta.OwnerID)
+	if err != nil {
+		return "", err
+	}
+	if err := PutObject(ctx, objects, bucket, key, region, meta); err != nil {
+		return "", err
+	}
+	return key, nil
+}
+
+// metadataValue reads name from S3 object metadata, whose keys the SDK
+// returns canonicalised.
+func metadataValue(md map[string]*string, name string) string {
+	for k, v := range md {
+		if strings.EqualFold(k, name) {
+			return aws.StringValue(v)
+		}
+	}
+	return ""
+}

spinifex/domains/ec2/instance/uefistore/image_test.go

@@ -0,0 +1,70 @@
+package uefistore_test
+
+import (
+	"testing"
+
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const imageID = "ami-0123456789abcdef0"
+
+// A launch only seeds from a store under the image's own prefix with a
+// firmware it can check; an AMI with no reference seeds from the template.
+func TestImageVarStoreFromAMI(t *testing.T) {
+	fw := ebsmetadata.UEFIFirmware{Arch: "x86_64", VarsSize: 4096, CodeSHA256: "c"}
+	own := uefistore.KeyPrefix(imageID) + "k"
+
+	got, err := uefistore.ImageVarStoreFromAMI(ebsmetadata.AMI{ImageID: imageID})
+	require.NoError(t, err)
+	assert.Nil(t, got)
+
+	got, err = uefistore.ImageVarStoreFromAMI(ebsmetadata.AMI{ImageID: imageID, UEFIData: &ebsmetadata.UEFIDataRef{Key: own, Firmware: fw}})
+	require.NoError(t, err)
+	assert.Equal(t, &uefistore.ImageVarStore{
+		ImageID: imageID, Key: own, Firmware: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: 4096, CodeSHA256: "c"},
+	}, got)
+
+	for name, ref := range map[string]ebsmetadata.UEFIDataRef{
+		"another image's key":   {Key: uefistore.KeyPrefix("ami-0fffffffffffffff0") + "k", Firmware: fw},
+		"an instance's key":     {Key: uefistore.KeyPrefix(instanceID) + "k", Firmware: fw},
+		"a nested key":          {Key: own + "/deeper", Firmware: fw},
+		"no firmware arch":      {Key: own, Firmware: ebsmetadata.UEFIFirmware{VarsSize: 4096}},
+		"no firmware VARS size": {Key: own, Firmware: ebsmetadata.UEFIFirmware{Arch: "x86_64"}},
+	} {
+		_, err := uefistore.ImageVarStoreFromAMI(ebsmetadata.AMI{ImageID: imageID, UEFIData: &ref})
+		require.ErrorIs(t, err, uefistore.ErrInvalidObject, name)
+	}
+}
+
+// A copy is a fresh key under the new owner holding the source's region, and
+// the source is left in place.
+func TestCopyObject_PutsTheRegionAtAFreshKeyUnderTheNewOwner(t *testing.T) {
+	objects := objectstore.NewMemoryObjectStore()
+	meta := testMeta(instanceID)
+	region := make([]byte, meta.Firmware.VarsSize)
+	copy(region, "variables")
+	src, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+	require.NoError(t, uefistore.PutObject(t.Context(), objects, testBucket, src, region, meta))
+
+	dst := testMeta(imageID)
+	dst.Source = uefistore.SourceInstance(instanceID)
+	key, err := uefistore.CopyObject(t.Context(), objects, testBucket, src, dst)
+	require.NoError(t, err)
+
+	keys, err := uefistore.ListObjects(t.Context(), objects, testBucket, imageID)
+	require.NoError(t, err)
+	assert.Equal(t, []string{key}, keys)
+	got, _, err := uefistore.ReadObject(t.Context(), objects, testBucket, key, meta.Firmware.VarsSize)
+	require.NoError(t, err)
+	assert.Equal(t, region, got)
+	_, _, err = uefistore.ReadObject(t.Context(), objects, testBucket, src, meta.Firmware.VarsSize)
+	require.NoError(t, err, "the source stays")
+
+	_, err = uefistore.CopyObject(t.Context(), objects, testBucket, uefistore.KeyPrefix(instanceID)+"gone", dst)
+	require.ErrorIs(t, err, uefistore.ErrObjectNotFound)
+}

spinifex/domains/ec2/instance/uefistore/object.go

@@ -0,0 +1,121 @@
+package uefistore
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+const (
+	// SourceTemplate is a store seeded from the launching node's VARS template.
+	SourceTemplate = "template"
+	// SourceWriteBack is a store written back from a node's live copy.
+	SourceWriteBack = "write-back"
+
+	// PutTimeout bounds one put of a store object. The object is a fixed-size
+	// region, so a put that takes longer is a wedged object store.
+	PutTimeout = 10 * time.Second
+)
+
+// ErrInvalidObject is a store object whose key, region or metadata cannot be
+// written as the owner's store.
+var ErrInvalidObject = errors.New("uefistore: invalid store object")
+
+// ObjectMeta is what a store object records about itself, as x-amz-meta-
+// entries, so its owner, writer and firmware are known from the object alone.
+type ObjectMeta struct {
+	OwnerID    string
+	WriterNode string
+	Firmware   FirmwareIdentity
+	CodePath   string
+	Source     string
+}
+
+// Metadata returns m as object metadata, keyed by the owner's kind.
+func (m ObjectMeta) Metadata() (map[string]*string, error) {
+	var ownerKey string
+	switch {
+	case strings.HasPrefix(m.OwnerID, "i-"):
+		ownerKey = "instance-id"
+	case strings.HasPrefix(m.OwnerID, "ami-"):
+		ownerKey = "image-id"
+	default:
+		return nil, fmt.Errorf("%w: owner %q is neither an instance nor an image", ErrInvalidObject, m.OwnerID)
+	}
+	switch {
+	case m.WriterNode == "":
+		return nil, fmt.Errorf("%w: %s names no writer node", ErrInvalidObject, m.OwnerID)
+	case m.Source == "":
+		return nil, fmt.Errorf("%w: %s names no source", ErrInvalidObject, m.OwnerID)
+	case m.Firmware.Arch == "" || m.Firmware.VarsSize <= 0 || m.Firmware.CodeSHA256 == "":
+		return nil, fmt.Errorf("%w: %s has an incomplete firmware identity", ErrInvalidObject, m.OwnerID)
+	}
+	return map[string]*string{
+		ownerKey:               aws.String(m.OwnerID),
+		"writer-node":          aws.String(m.WriterNode),
+		"firmware-arch":        aws.String(m.Firmware.Arch),
+		"firmware-vars-size":   aws.String(strconv.FormatInt(m.Firmware.VarsSize, 10)),
+		"firmware-code-sha256": aws.String(m.Firmware.CodeSHA256),
+		"firmware-code-path":   aws.String(m.CodePath),
+		"source":               aws.String(m.Source),
+	}, nil
+}
+
+// NewObjectKey returns a fresh key under ownerID's prefix, named by a random
+// version 4 UUID, so no write ever replaces an object a record points at.
+func NewObjectKey(ownerID string) (string, error) {
+	var b [16]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", fmt.Errorf("generate UEFI store key for %s: %w", ownerID, err)
+	}
+	b[6] = b[6]&0x0f | 0x40
+	b[8] = b[8]&0x3f | 0x80
+	return fmt.Sprintf("%s%x-%x-%x-%x-%x", KeyPrefix(ownerID), b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]), nil
+}
+
+// PutObject writes region gzip-compressed to key with meta, within
+// PutTimeout. The region must be exactly the firmware's VARS size, because
+// pflash refuses any other.
+func PutObject(ctx context.Context, objects objectstore.ObjectStore, bucket, key string, region []byte, meta ObjectMeta) error {
+	if !ownsKey(meta.OwnerID, key) {
+		return fmt.Errorf("%w: key %q is not under %s", ErrInvalidObject, key, KeyPrefix(meta.OwnerID))
+	}
+	if int64(len(region)) != meta.Firmware.VarsSize {
+		return fmt.Errorf("%w: %s is a %d-byte region under firmware with a %d-byte VARS region",
+			ErrInvalidObject, key, len(region), meta.Firmware.VarsSize)
+	}
+	md, err := meta.Metadata()
+	if err != nil {
+		return err
+	}
+	var body bytes.Buffer
+	zw := gzip.NewWriter(&body)
+	if _, err := zw.Write(region); err != nil {
+		return fmt.Errorf("compress UEFI store %s: %w", key, err)
+	}
+	if err := zw.Close(); err != nil {
+		return fmt.Errorf("compress UEFI store %s: %w", key, err)
+	}
+
+	putCtx, cancel := context.WithTimeout(ctx, PutTimeout)
+	defer cancel()
+	if _, err := objects.PutObject(putCtx, &s3.PutObjectInput{
+		Bucket:   aws.String(bucket),
+		Key:      aws.String(key),
+		Body:     bytes.NewReader(body.Bytes()),
+		Metadata: md,
+	}); err != nil {
+		return fmt.Errorf("put UEFI store %s: %w", key, err)
+	}
+	return nil
+}

spinifex/domains/ec2/instance/uefistore/object_test.go

@@ -0,0 +1,164 @@
+package uefistore_test
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"io"
+	"regexp"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const testBucket = "instances"
+
+func testMeta(owner string) uefistore.ObjectMeta {
+	return uefistore.ObjectMeta{
+		OwnerID:    owner,
+		WriterNode: "node-1",
+		Firmware:   uefistore.FirmwareIdentity{Arch: "arm64", VarsSize: 1 << 16, CodeSHA256: "abc123"},
+		CodePath:   "/usr/share/AAVMF/AAVMF_CODE.fd",
+		Source:     uefistore.SourceTemplate,
+	}
+}
+
+var uuidV4 = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
+
+// Every key is a fresh version 4 UUID directly under the owner's prefix, so
+// a write never lands on an object a record already points at.
+func TestNewObjectKey_IsAFreshUUIDUnderTheOwnersPrefix(t *testing.T) {
+	seen := map[string]bool{}
+	for range 64 {
+		key, err := uefistore.NewObjectKey(instanceID)
+		require.NoError(t, err)
+		name, ok := bytes.CutPrefix([]byte(key), []byte(uefistore.KeyPrefix(instanceID)))
+		require.True(t, ok, key)
+		assert.Regexp(t, uuidV4, string(name))
+		assert.False(t, seen[key], "key %s repeated", key)
+		seen[key] = true
+	}
+}
+
+func TestObjectMeta_Metadata_NamesTheOwnerByKind(t *testing.T) {
+	md, err := testMeta(instanceID).Metadata()
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{
+		"instance-id":          instanceID,
+		"writer-node":          "node-1",
+		"firmware-arch":        "arm64",
+		"firmware-vars-size":   "65536",
+		"firmware-code-sha256": "abc123",
+		"firmware-code-path":   "/usr/share/AAVMF/AAVMF_CODE.fd",
+		"source":               "template",
+	}, aws.StringValueMap(md))
+
+	img := testMeta("ami-0123456789abcdef0")
+	img.Source = "image:ami-0fedcba9876543210"
+	md, err = img.Metadata()
+	require.NoError(t, err)
+	assert.Equal(t, "ami-0123456789abcdef0", aws.StringValue(md["image-id"]))
+	assert.NotContains(t, md, "instance-id")
+	assert.Equal(t, "image:ami-0fedcba9876543210", aws.StringValue(md["source"]))
+}
+
+func TestObjectMeta_Metadata_RefusesAnIncompleteObject(t *testing.T) {
+	tests := map[string]func(*uefistore.ObjectMeta){
+		"unknown owner":  func(m *uefistore.ObjectMeta) { m.OwnerID = "vol-1" },
+		"no writer":      func(m *uefistore.ObjectMeta) { m.WriterNode = "" },
+		"no source":      func(m *uefistore.ObjectMeta) { m.Source = "" },
+		"no arch":        func(m *uefistore.ObjectMeta) { m.Firmware.Arch = "" },
+		"no vars size":   func(m *uefistore.ObjectMeta) { m.Firmware.VarsSize = 0 },
+		"no code sha256": func(m *uefistore.ObjectMeta) { m.Firmware.CodeSHA256 = "" },
+	}
+	for name, mutate := range tests {
+		t.Run(name, func(t *testing.T) {
+			m := testMeta(instanceID)
+			mutate(&m)
+			_, err := m.Metadata()
+			require.ErrorIs(t, err, uefistore.ErrInvalidObject)
+		})
+	}
+}
+
+// The 64 MiB arm64 region is mostly erased flash, so it is stored compressed
+// and decompresses to the exact region.
+func TestPutObject_StoresTheRegionCompressed(t *testing.T) {
+	objects := objectstore.NewMemoryObjectStore()
+	meta := testMeta(instanceID)
+	region := bytes.Repeat([]byte{0xff}, int(meta.Firmware.VarsSize))
+	copy(region, "BootOrder")
+	key, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+
+	require.NoError(t, uefistore.PutObject(t.Context(), objects, testBucket, key, region, meta))
+
+	out, err := objects.GetObject(t.Context(), &s3.GetObjectInput{Bucket: aws.String(testBucket), Key: aws.String(key)})
+	require.NoError(t, err)
+	body, err := io.ReadAll(out.Body)
+	require.NoError(t, err)
+	assert.Less(t, len(body), len(region)/10, "the erased region compresses")
+	gz, err := gzip.NewReader(bytes.NewReader(body))
+	require.NoError(t, err)
+	got, err := io.ReadAll(gz)
+	require.NoError(t, err)
+	assert.Equal(t, region, got)
+}
+
+// pflash needs a byte-exact region, and a key outside the owner's prefix is
+// one no store record owns; either is refused before anything is put.
+func TestPutObject_RefusesWhatTheOwnerCannotBoot(t *testing.T) {
+	meta := testMeta(instanceID)
+	good, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+	other, err := uefistore.NewObjectKey("i-0fedcba9876543210")
+	require.NoError(t, err)
+	tests := []struct {
+		name   string
+		key    string
+		region []byte
+	}{
+		{name: "short region", key: good, region: make([]byte, meta.Firmware.VarsSize-1)},
+		{name: "long region", key: good, region: make([]byte, meta.Firmware.VarsSize+1)},
+		{name: "another owner's key", key: other, region: make([]byte, meta.Firmware.VarsSize)},
+		{name: "nested key", key: uefistore.KeyPrefix(instanceID) + "a/b", region: make([]byte, meta.Firmware.VarsSize)},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			objects := objectstore.NewMemoryObjectStore()
+			err := uefistore.PutObject(t.Context(), objects, testBucket, tt.key, tt.region, meta)
+			require.ErrorIs(t, err, uefistore.ErrInvalidObject)
+			out, listErr := objects.ListObjectsV2(t.Context(), &s3.ListObjectsV2Input{Bucket: aws.String(testBucket)})
+			require.NoError(t, listErr)
+			assert.Empty(t, out.Contents)
+		})
+	}
+}
+
+type failingPut struct {
+	*objectstore.MemoryObjectStore
+}
+
+func (failingPut) PutObject(ctx context.Context, _ *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
+	if _, ok := ctx.Deadline(); !ok {
+		return nil, errors.New("put had no deadline")
+	}
+	return nil, errors.New("predastore unreachable")
+}
+
+// A put runs under a deadline, so a wedged object store fails the write
+// rather than holding the caller, and its error reaches the caller.
+func TestPutObject_IsBoundedAndReportsAFailedPut(t *testing.T) {
+	key, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+	meta := testMeta(instanceID)
+	err = uefistore.PutObject(t.Context(), failingPut{objectstore.NewMemoryObjectStore()}, testBucket, key, make([]byte, meta.Firmware.VarsSize), meta)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "predastore unreachable")
+}

spinifex/domains/ec2/instance/uefistore/prune.go

@@ -0,0 +1,73 @@
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+// PruneMinAge is how old an object must be before a prune may delete it, so a
+// write-back whose pointer move has not landed yet never loses its object.
+const PruneMinAge = 10 * time.Minute
+
+// PruneResult is what one prune of a prefix did.
+type PruneResult struct {
+	Deleted int
+	// Young counts objects other than keep that were too new to delete, so the
+	// caller prunes the prefix again later.
+	Young int
+}
+
+// PruneObjects deletes every object under ownerID's prefix except keep that was
+// last modified before cutoff. keep must be the key the store record points at;
+// an empty or foreign keep deletes nothing. An object with no modification time
+// counts as young.
+func PruneObjects(ctx context.Context, objects objectstore.ObjectStore, bucket, ownerID, keep string, cutoff time.Time) (PruneResult, error) {
+	var res PruneResult
+	if err := checkOwnedKey(ownerID, keep); err != nil {
+		return res, fmt.Errorf("uefistore: refusing to prune %s: %w", ownerID, err)
+	}
+	contents, _, err := objectstore.ListAll(ctx, objects, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String(KeyPrefix(ownerID)),
+	})
+	if err != nil {
+		return res, fmt.Errorf("uefistore: list %s: %w", KeyPrefix(ownerID), err)
+	}
+	for _, obj := range contents {
+		key := aws.StringValue(obj.Key)
+		if key == keep {
+			continue
+		}
+		if obj.LastModified == nil || !obj.LastModified.Before(cutoff) {
+			res.Young++
+			continue
+		}
+		_, err := objects.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
+		if err != nil && !objectstore.IsNoSuchKeyError(err) {
+			return res, fmt.Errorf("uefistore: delete %s: %w", key, err)
+		}
+		res.Deleted++
+	}
+	return res, nil
+}
+
+// checkOwnedKey fails a key that is empty or not one object under ownerID's
+// prefix.
+func checkOwnedKey(ownerID, key string) error {
+	if !validOwner(ownerID) {
+		return fmt.Errorf("%q is not an instance or image ID", ownerID)
+	}
+	if key == "" {
+		return errors.New("no key to keep")
+	}
+	if !ownsKey(ownerID, key) {
+		return fmt.Errorf("%s is not under %s", key, KeyPrefix(ownerID))
+	}
+	return nil
+}

spinifex/domains/ec2/instance/uefistore/prune_test.go

@@ -0,0 +1,84 @@
+package uefistore_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// datedStore lists each object with the modification time the test set; a
+// key it has none for is listed with none.
+type datedStore struct {
+	*objectstore.MemoryObjectStore
+
+	modified map[string]time.Time
+}
+
+func (s *datedStore) ListObjectsV2(ctx context.Context, in *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	out, err := s.MemoryObjectStore.ListObjectsV2(ctx, in)
+	if err != nil {
+		return nil, err
+	}
+	for _, obj := range out.Contents {
+		obj.LastModified = nil
+		if at, ok := s.modified[aws.StringValue(obj.Key)]; ok {
+			obj.LastModified = aws.Time(at)
+		}
+	}
+	return out, nil
+}
+
+// A prune keeps the key it is told to and every object not older than the
+// cutoff, counting those it kept for age, and an undated object counts as
+// young (ADR-0003:S7).
+func TestS7_PruneObjectsKeepsThePointerAndEveryYoungObject(t *testing.T) {
+	const owner = "i-0123456789abcdef0"
+	prefix := uefistore.KeyPrefix(owner)
+	cutoff := time.Unix(1700000000, 0)
+	keep, stale, young, undated, other := prefix+"keep", prefix+"stale", prefix+"young", prefix+"undated", uefistore.KeyPrefix("i-0fedcba9876543210")+"stale"
+	store := &datedStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), modified: map[string]time.Time{
+		keep:  cutoff.Add(-time.Hour),
+		stale: cutoff.Add(-time.Second),
+		young: cutoff,
+		other: cutoff.Add(-time.Hour),
+	}}
+	for _, k := range []string{keep, stale, young, undated, other} {
+		putObject(t, store, k)
+	}
+
+	res, err := uefistore.PruneObjects(t.Context(), store, bucket, owner, keep, cutoff)
+
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.PruneResult{Deleted: 1, Young: 2}, res)
+	assert.ElementsMatch(t, []string{keep, young, undated}, keysUnder(t, store, prefix),
+		"ADR-0003:S7: a prune must keep the pointed object and every object younger than the cutoff")
+	assert.Equal(t, []string{other}, keysUnder(t, store, uefistore.KeyPrefix("i-0fedcba9876543210")), "another owner's prefix is untouched")
+}
+
+// With no key to keep, or one outside the owner's prefix, a prune cannot tell
+// the pointer from a leftover, so it deletes nothing (ADR-0003:S7).
+func TestS7_PruneObjectsWithoutAKeyToKeepDeletesNothing(t *testing.T) {
+	const owner = "i-0123456789abcdef0"
+	prefix := uefistore.KeyPrefix(owner)
+	store := &datedStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), modified: map[string]time.Time{
+		prefix + "stale": time.Unix(1, 0),
+	}}
+	putObject(t, store, prefix+"stale")
+
+	for _, keep := range []string{"", uefistore.KeyPrefix("i-0fedcba9876543210") + "stale", prefix + "nested/key"} {
+		_, err := uefistore.PruneObjects(t.Context(), store, bucket, owner, keep, time.Now())
+		require.Error(t, err, keep)
+	}
+	_, err := uefistore.PruneObjects(t.Context(), store, bucket, "i-*", prefix+"stale", time.Now())
+	require.Error(t, err)
+
+	assert.Equal(t, []string{prefix + "stale"}, keysUnder(t, store, prefix),
+		"ADR-0003:S7: a prune with no valid pointer to keep must delete nothing")
+}

spinifex/domains/ec2/instance/uefistore/uefistore.go

@@ -1,6 +1,6 @@
 // Package uefistore holds the types that describe an instance's UEFI variable
-// store and the bucket of store records that carry its deletion and write-back
-// obligations past the life of the instance record.
+// store, the bucket of store records that carry its pointer and deletion
+// obligation past the life of the instance record, and the claim that fences it.
 package uefistore
 
 import (
@@ -53,22 +53,27 @@ type FirmwareIdentity struct {
 // UEFIStoreRecord is the durable owner of one instance's variable store
 // objects. It outlives the instance record, so an obligation recorded here is
 // not lost when a terminated record expires.
+//
+// Pointer names the only version of the store the record keeps, the object the
+// instance boots from, and only a writer whose claim token has generation
+// Generation may move it.
 type UEFIStoreRecord struct {
-	SchemaVersion uint16         `json:"schema_version"`
-	InstanceID    string         `json:"instance_id"`
-	AccountID     string         `json:"account_id"`
-	State         string         `json:"state"`
-	WriteBack     *UEFIWriteBack `json:"write_back,omitempty"`
+	SchemaVersion uint16           `json:"schema_version"`
+	InstanceID    string           `json:"instance_id"`
+	AccountID     string           `json:"account_id"`
+	State         string           `json:"state"`
+	Generation    uint64           `json:"generation"`
+	Pointer       *UEFIVarStoreRef `json:"pointer,omitempty"`
 }
 
-// UEFIWriteBack is a write-back that failed and is still owed. Key is the
-// object every retry puts to, so retries leave no extra objects; SeededFrom is
-// the pointer Node seeded from, which a retry must still find on the record.
-type UEFIWriteBack struct {
-	Node       string    `json:"node"`
-	Key        string    `json:"key"`
-	SeededFrom string    `json:"seeded_from"`
-	Since      time.Time `json:"since"`
+// UEFIVarStoreRef names the store object an instance launches from, with the
+// firmware it was written under, so a launch can check it without reading the
+// object's metadata.
+type UEFIVarStoreRef struct {
+	Key        string           `json:"key"`
+	Firmware   FirmwareIdentity `json:"firmware"`
+	WriterNode string           `json:"writer_node"`
+	WrittenAt  time.Time        `json:"written_at"`
 }
 
 // KeyPrefix is the object-store prefix every store object of ownerID (an
@@ -83,8 +88,8 @@ func ownsKey(ownerID, key string) bool {
 	return ok && name != "" && !strings.Contains(name, "/")
 }
 
-// validate checks rec against key. A write-back key outside the instance's own
-// prefix is refused because a superseded obligation deletes that key.
+// validate checks rec against key. The pointer must name one object directly
+// under the instance's own prefix, because a deletion acts on it.
 func (rec *UEFIStoreRecord) validate(key string) error {
 	switch {
 	case rec.SchemaVersion > RecordSchemaVersion:
@@ -98,29 +103,32 @@ func (rec *UEFIStoreRecord) validate(key string) error {
 		return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
 	case rec.State != StateLive && rec.State != StateDeleting:
 		return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
-	case rec.State == StateDeleting && rec.WriteBack != nil:
-		// A write-back retry puts an object, which would land behind the delete.
-		return fmt.Errorf("%w: %s is deleting with a write-back still owed", ErrInvalidRecord, key)
 	}
-	if wb := rec.WriteBack; wb != nil {
-		switch {
-		case wb.Node == "":
-			return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
-		case wb.Since.IsZero():
-			return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
-		case !ownsKey(rec.InstanceID, wb.Key):
-			return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
-		case !ownsKey(rec.InstanceID, wb.SeededFrom):
-			return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
+	if p := rec.Pointer; p != nil {
+		if err := p.validate(rec.InstanceID); err != nil {
+			return fmt.Errorf("%w: %s pointer: %w", ErrInvalidRecord, key, err)
 		}
 	}
 	return nil
 }
 
+// validate checks ref as a pointer of instanceID's store.
+func (ref *UEFIVarStoreRef) validate(instanceID string) error {
+	switch {
+	case !ownsKey(instanceID, ref.Key):
+		return fmt.Errorf("key %q is not under %s", ref.Key, KeyPrefix(instanceID))
+	case ref.Firmware.Arch == "" || ref.Firmware.VarsSize <= 0:
+		return fmt.Errorf("key %q names no firmware arch and VARS size", ref.Key)
+	case ref.WriterNode == "":
+		return fmt.Errorf("key %q names no writer node", ref.Key)
+	}
+	return nil
+}
+
 func bucketConfig() kvstore.Config {
 	return kvstore.Config{
 		Name:        BucketName,
-		Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
+		Description: "Spinifex UEFI variable store records (pointers and deletion obligations)",
 		History:     1,
 		// Left false: a lost bucket has lost obligations, and an empty
 		// recreated one would report every store as having none.
@@ -172,6 +180,19 @@ func (s *Store) Get(ctx context.Context, instanceID string) (*UEFIStoreRecord, u
 	return rec, rev, nil
 }
 
+// LeaderGet is Get answered by the stream leader, for a caller that deletes
+// objects by what the record does not point at.
+func (s *Store) LeaderGet(ctx context.Context, instanceID string) (*UEFIStoreRecord, uint64, error) {
+	rec, rev, err := s.kv.LeaderGet(ctx, instanceID)
+	if err != nil {
+		return nil, 0, err
+	}
+	if err := rec.validate(instanceID); err != nil {
+		return nil, 0, err
+	}
+	return rec, rev, nil
+}
+
 // Create writes a new record, which must be live: a store is recorded before
 // its first object is written. An existing record is kvstore.ErrExists. Every
 // write stamps rec with RecordSchemaVersion.

spinifex/domains/ec2/instance/uefistore/uefistore_test.go

@@ -11,7 +11,6 @@ import (
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -31,15 +30,25 @@ func liveRecord() *uefistore.UEFIStoreRecord {
 	return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
 }
 
-func writeBack() *uefistore.UEFIWriteBack {
-	return &uefistore.UEFIWriteBack{
-		Node:       "node-2",
-		Key:        uefistore.KeyPrefix(instanceID) + "b",
-		SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
-		Since:      time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
+func pointer(name string) *uefistore.UEFIVarStoreRef {
+	return &uefistore.UEFIVarStoreRef{
+		Key:        keyOf(name),
+		Firmware:   uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: 540672, CodeSHA256: "c0de"},
+		WriterNode: "node-2",
+		WrittenAt:  time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
 	}
 }
 
+func keyOf(name string) string { return uefistore.KeyPrefix(instanceID) + name }
+
+// pointedRecord is a live record stamped at generation 7 that points at "b".
+func pointedRecord() *uefistore.UEFIStoreRecord {
+	rec := liveRecord()
+	rec.Generation = 7
+	rec.Pointer = pointer("b")
+	return rec
+}
+
 func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
 	t.Helper()
 	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
@@ -60,15 +69,14 @@ func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
 	assert.Equal(t, int64(1), status.History())
 	assert.Zero(t, status.TTL(), "a TTL would expire deletion obligations before they are settled")
 
-	v, err := kvutil.ReadVersion(t.Context(), kv)
+	v, err := kvstore.ReadVersion(t.Context(), kv)
 	require.NoError(t, err)
 	assert.Equal(t, uefistore.BucketVersion, v)
 }
 
 func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
 	s, _ := openStore(t)
-	rec := liveRecord()
-	rec.WriteBack = writeBack()
+	rec := pointedRecord()
 
 	rev, err := s.Create(t.Context(), rec)
 	require.NoError(t, err)
@@ -83,9 +91,7 @@ func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
 // The wire names are what every release reads, so they are pinned.
 func TestRecord_WireShape(t *testing.T) {
 	s, js := openStore(t)
-	rec := liveRecord()
-	rec.WriteBack = writeBack()
-	_, err := s.Create(t.Context(), rec)
+	_, err := s.Create(t.Context(), pointedRecord())
 	require.NoError(t, err)
 
 	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
@@ -97,11 +103,12 @@ func TestRecord_WireShape(t *testing.T) {
 		"instance_id": "i-0123456789abcdef0",
 		"account_id": "111122223333",
 		"state": "live",
-		"write_back": {
-			"node": "node-2",
+		"generation": 7,
+		"pointer": {
 			"key": "uefi/i-0123456789abcdef0/b",
-			"seeded_from": "uefi/i-0123456789abcdef0/a",
-			"since": "2026-11-03T01:02:03Z"
+			"firmware": {"arch": "x86_64", "vars_size": 540672, "code_sha256": "c0de"},
+			"writer_node": "node-2",
+			"written_at": "2026-11-03T01:02:03Z"
 		}
 	}`, string(entry.Value()))
 }
@@ -145,8 +152,7 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
 	first, err := s.Create(t.Context(), liveRecord())
 	require.NoError(t, err)
 
-	moved := liveRecord()
-	moved.WriteBack = writeBack()
+	moved := pointedRecord()
 	second, err := s.CompareAndSet(t.Context(), moved, first)
 	require.NoError(t, err)
 	assert.Greater(t, second, first)
@@ -160,7 +166,7 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, second, rev)
 	assert.Equal(t, uefistore.StateLive, got.State)
-	assert.Equal(t, writeBack(), got.WriteBack)
+	assert.Equal(t, pointer("b"), got.Pointer)
 }
 
 // Two actors that read the same revision race to change it; exactly one may
@@ -180,12 +186,11 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
 	)
 	for i := range writers {
 		wg.Go(func() {
-			rec := liveRecord()
-			rec.WriteBack = writeBack()
-			rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
+			rec := pointedRecord()
+			rec.Pointer.WriterNode = fmt.Sprintf("node-%d", i)
 			if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
 				mu.Lock()
-				wins = append(wins, rec.WriteBack.Node)
+				wins = append(wins, rec.Pointer.WriterNode)
 				mu.Unlock()
 			} else {
 				assert.ErrorIs(t, err, kvstore.ErrConflict)
@@ -197,7 +202,7 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
 	require.Len(t, wins, 1)
 	got, _, err := s.Get(t.Context(), instanceID)
 	require.NoError(t, err)
-	assert.Equal(t, wins[0], got.WriteBack.Node)
+	assert.Equal(t, wins[0], got.Pointer.WriterNode)
 }
 
 func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
@@ -218,29 +223,20 @@ func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
 	require.ErrorIs(t, err, kvstore.ErrNotFound)
 }
 
-// A superseded obligation deletes its recorded key, so a key outside the
+// A deletion acts on the key the record points at, so a key outside the
 // instance's own prefix must never reach the bucket.
 func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
+	other := uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
 	cases := map[string]func(*uefistore.UEFIStoreRecord){
-		"unknown state":            func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
-		"not an instance ID":       func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
-		"write-back with no node":  func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
-		"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
-		"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
-		},
-		"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
-		},
-		"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
-		},
-		"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
-		},
-		"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
-		},
+		"unknown state":                   func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
+		"not an instance ID":              func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
+		"pointer under another instance":  func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = other },
+		"pointer under an image":          func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b" },
+		"pointer is the prefix itself":    func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix(instanceID) },
+		"pointer nested below the prefix": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = keyOf("x/y") },
+		"pointer with no firmware arch":   func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.Arch = "" },
+		"pointer with no VARS size":       func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.VarsSize = 0 },
+		"pointer with no writer node":     func(r *uefistore.UEFIStoreRecord) { r.Pointer.WriterNode = "" },
 	}
 	for name, mutate := range cases {
 		t.Run(name, func(t *testing.T) {
@@ -248,8 +244,7 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 			rev, err := s.Create(t.Context(), liveRecord())
 			require.NoError(t, err)
 
-			rec := liveRecord()
-			rec.WriteBack = writeBack()
+			rec := pointedRecord()
 			mutate(rec)
 			_, err = s.CompareAndSet(t.Context(), rec, rev)
 			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
@@ -257,7 +252,7 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 			got, gotRev, err := s.Get(t.Context(), instanceID)
 			require.NoError(t, err)
 			assert.Equal(t, rev, gotRev, "an invalid record must not be written")
-			assert.Nil(t, got.WriteBack)
+			assert.Nil(t, got.Pointer)
 		})
 	}
 }
@@ -266,11 +261,11 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 // is reported rather than decoded into something a reaper would act on.
 func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
 	cases := map[string]string{
-		"newer schema":       `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
-		"no schema":          `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
-		"another instance":   `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
-		"unknown state":      `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
-		"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
+		"newer schema":     `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"no schema":        `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"another instance": `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
+		"unknown state":    `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
+		"foreign pointer":  `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","generation":3,"pointer":{"key":"uefi/i-0fedcba9876543210/b","firmware":{"arch":"x86_64","vars_size":540672},"writer_node":"n"}}`,
 	}
 	for name, body := range cases {
 		t.Run(name, func(t *testing.T) {

spinifex/domains/ec2/instance/volume_held_elsewhere_test.go

@@ -102,3 +102,38 @@ func TestStartPaths_ReportAVolumeHeldElsewhereByCode(t *testing.T) {
 		assert.Equal(t, awserrors.ErrorIncorrectState, awserrors.ValidErrorCodeFromError(err))
 	})
 }
+
+// uefiClaimHeldMounter fails the launch the way a start does while a live
+// node holds the instance's UEFI variable store.
+type uefiClaimHeldMounter struct{ raceVolumeMounter }
+
+func (uefiClaimHeldMounter) Mount(context.Context, *vm.VM) error {
+	return fmt.Errorf("%w: node-a holds i-uefi: %w", vm.ErrUEFIClaimHeld, vm.ErrMountRetryable)
+}
+
+// A held UEFI store is a capacity refusal on both start paths, so the caller
+// tries another node rather than reporting a fault.
+func TestStartPaths_ReportAHeldUEFIStoreAsCapacity(t *testing.T) {
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	types, _ := defaultPrepareInstanceTypes()
+
+	t.Run("claimed relaunch", func(t *testing.T) {
+		mgr := vm.NewManager()
+		mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
+		svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
+
+		err := svc.launchClaimedInstance(t.Context(), &vm.VM{ID: "i-uefi-claim", AccountID: "acc", InstanceType: "t3.micro", Status: vm.StatePending}, func() {})
+		assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
+	})
+
+	t.Run("live owner", func(t *testing.T) {
+		const id = "i-uefi-live"
+		mgr := mgrWith(map[string]*vm.VM{id: {ID: id, Status: vm.StateStopped, InstanceType: "t3.micro"}})
+		mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
+		v, _ := mgr.Get(id)
+		svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
+
+		err := svc.StartInstance(t.Context(), v, ec2v1.EC2InstanceCommand{ID: id})
+		assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
+	})
+}

spinifex/domains/ec2/launchtemplate/account_cleanup.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -33,7 +33,7 @@ type AccountCleanup struct {
 // Unreadable headers, their bodies and unrecognised entries are left for the
 // owner scan to report. A listing failure is returned and nothing is removed.
 func (s *LaunchTemplateServiceImpl) CleanupAccountResidue(ctx context.Context, accountID string, settledBefore time.Time) (*AccountCleanup, error) {
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list launch-template keys: %w", err)
 	}
@@ -145,7 +145,7 @@ type AccountTemplateDeletes struct {
 // it cannot read or decode is left for the owner scan to report. A listing
 // failure is returned and nothing is deleted.
 func (s *LaunchTemplateServiceImpl) DeleteLiveTemplates(ctx context.Context, accountID string, referenced func(id, name string) string) (*AccountTemplateDeletes, error) {
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list launch-template keys: %w", err)
 	}

spinifex/domains/ec2/launchtemplate/account_scan.go

@@ -9,7 +9,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -37,7 +37,7 @@ type AccountScan struct {
 // Headers and name-index entries are read; version bodies are classified from
 // their keys alone, so template data and user data are never loaded.
 func (s *LaunchTemplateServiceImpl) ScanAccount(ctx context.Context, accountID string) (*AccountScan, error) {
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list launch-template keys: %w", err)
 	}

spinifex/domains/ec2/launchtemplate/account_scan_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit/readonlystore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -45,7 +45,7 @@ func TestScanAccountClassifiesLaunchTemplateRecords(t *testing.T) {
 	// A header removed alone, as a delete interrupted after it leaves the rest.
 	deleted := createWithUserData(t, svc, "deleted")
 	inner := svc.kv
-	require.NoError(t, inner.Delete(t.Context(), kvutil.AccountKey(testAccountID, deleted)))
+	require.NoError(t, inner.Delete(t.Context(), kvstore.AccountKey(testAccountID, deleted)))
 
 	// A create whose header write and clean-up both fail.
 	svc.kv = &faultyKV{KeyValue: inner,
@@ -59,16 +59,16 @@ func TestScanAccountClassifiesLaunchTemplateRecords(t *testing.T) {
 	require.Error(t, err)
 	svc.kv = inner
 
-	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0badbadbadbadbad0"), "{not json")
-	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0badbadbadbadbad0.v1"), `{"version_number":1}`)
-	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0mismatch00000000"), `{"launch_template_id":"lt-0other"}`)
-	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0unreadable000000"), `{"launch_template_id":"lt-0unreadable000000"}`)
+	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0badbadbadbadbad0"), "{not json")
+	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0badbadbadbadbad0.v1"), `{"version_number":1}`)
+	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0mismatch00000000"), `{"launch_template_id":"lt-0other"}`)
+	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0unreadable000000"), `{"launch_template_id":"lt-0unreadable000000"}`)
 	putRaw(t, inner, nameKey(testAccountID, "unreadable"), "lt-0unreadable000000")
-	putRaw(t, inner, kvutil.AccountKey(testAccountID, "lt-0bad.vx"), "{}")
+	putRaw(t, inner, kvstore.AccountKey(testAccountID, "lt-0bad.vx"), "{}")
 
 	recorder := &readRecordingKV{KeyValue: readonlystore.NewKV(t, inner)}
 	svc.kv = &faultyKV{KeyValue: recorder, failGet: func(key string) bool {
-		return key == kvutil.AccountKey(testAccountID, "lt-0unreadable000000")
+		return key == kvstore.AccountKey(testAccountID, "lt-0unreadable000000")
 	}}
 	scan, err := svc.ScanAccount(context.Background(), testAccountID)
 	require.NoError(t, err)

spinifex/domains/ec2/launchtemplate/account_termination_current_behaviour_test.go

@@ -16,7 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -159,7 +159,7 @@ func TestCurrentBehaviour_AccountTermination_LaunchTemplateStorageLayout(t *test
 
 	header := testAccountID + "." + id
 	name := testAccountID + ".name.776562283129" // hex of "web(1)"
-	assert.Equal(t, sortedKeys(header, header+".v1", header+".v2", name, kvutil.VersionKey), storedKeys(t, svc.kv))
+	assert.Equal(t, sortedKeys(header, header+".v1", header+".v2", name, kvstore.VersionKey), storedKeys(t, svc.kv))
 
 	entry, err := svc.kv.Get(t.Context(), name)
 	require.NoError(t, err)
@@ -223,13 +223,13 @@ func TestAccountTermination_DeleteLaunchTemplateKeepsTheTemplateWhenAStepFails(t
 			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 
 			svc.kv = inner
-			assert.Equal(t, sortedKeys(append(tc.left(header, nameIdx), kvutil.VersionKey)...), storedKeys(t, inner))
+			assert.Equal(t, sortedKeys(append(tc.left(header, nameIdx), kvstore.VersionKey)...), storedKeys(t, inner))
 			assert.Equal(t, []string{id}, describedTemplateIDs(t, svc, testAccountID))
 
 			out, err := svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)
 			require.NoError(t, err)
 			assert.Equal(t, id, aws.StringValue(out.LaunchTemplate.LaunchTemplateId))
-			assert.Equal(t, []string{kvutil.VersionKey}, storedKeys(t, inner))
+			assert.Equal(t, []string{kvstore.VersionKey}, storedKeys(t, inner))
 			assert.Empty(t, describedTemplateIDs(t, svc, testAccountID))
 		})
 	}
@@ -260,7 +260,7 @@ func TestAccountTermination_DeleteLaunchTemplateRestartsWhenTheHeaderChanges(t *
 	_, err = svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)
 	require.NoError(t, err)
 	assert.True(t, raced)
-	assert.Equal(t, []string{kvutil.VersionKey}, storedKeys(t, inner))
+	assert.Equal(t, []string{kvstore.VersionKey}, storedKeys(t, inner))
 }
 
 // A create that fails after claiming the name removes what it wrote, so no
@@ -287,7 +287,7 @@ func TestAccountTermination_FailedCreateRemovesWhatItWrote(t *testing.T) {
 			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 
 			svc.kv = inner
-			assert.Equal(t, []string{kvutil.VersionKey}, storedKeys(t, inner))
+			assert.Equal(t, []string{kvstore.VersionKey}, storedKeys(t, inner))
 			retried := createTemplate(t, svc, "web", "t3.micro")
 			assert.Equal(t, []string{aws.StringValue(retried.LaunchTemplateId)}, describedTemplateIDs(t, svc, testAccountID))
 		})
@@ -299,7 +299,7 @@ func TestAccountTermination_FailedCreateRemovesWhatItWrote(t *testing.T) {
 // failure is an internal error, never a missing template.
 func TestCurrentBehaviour_AccountTermination_DescribeLaunchTemplatesHidesUnreadableHeaders(t *testing.T) {
 	const brokenID = "lt-0badbadbadbadbad0"
-	brokenHeader := kvutil.AccountKey(testAccountID, brokenID)
+	brokenHeader := kvstore.AccountKey(testAccountID, brokenID)
 	tests := []struct {
 		name       string
 		seed       func(t *testing.T, svc *LaunchTemplateServiceImpl)
@@ -350,7 +350,7 @@ func TestAccountCleanup_KeepsTemplatesAndRemovesSettledResidue(t *testing.T) {
 	require.NoError(t, err)
 	orphanBody := versionKey(testAccountID, "lt-0orphan000000000", 1)
 	orphanName := nameKey(testAccountID, "gone")
-	brokenHeader := kvutil.AccountKey(testAccountID, "lt-0broken000000000")
+	brokenHeader := kvstore.AccountKey(testAccountID, "lt-0broken000000000")
 	brokenBody := versionKey(testAccountID, "lt-0broken000000000", 1)
 	brokenName := nameKey(testAccountID, "broken")
 	for key, value := range map[string]string{
@@ -384,7 +384,7 @@ func TestAccountCleanup_KeepsTemplatesAndRemovesSettledResidue(t *testing.T) {
 func TestAccountCleanup_InterruptedTemplateDeleteIsLeftToThePublicRetry(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	kv := svc.kv
 	svc.kv = &faultyKV{KeyValue: kv, failDelete: func(key string) bool { return key == header }}
 	_, err := svc.DeleteLaunchTemplate(context.Background(), &ec2.DeleteLaunchTemplateInput{LaunchTemplateId: aws.String(id)}, testAccountID)

spinifex/domains/ec2/launchtemplate/service_impl.go

@@ -19,7 +19,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -64,7 +64,7 @@ func NewLaunchTemplateServiceImplWithNATS(ctx context.Context, cfg *config.Confi
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketLaunchTemplates, 10)
+	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketLaunchTemplates, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketLaunchTemplates, err)
 	}
@@ -87,24 +87,24 @@ func NewLaunchTemplateServiceImplWithNATS(ctx context.Context, cfg *config.Confi
 // Launch template names allow characters such as parentheses that are not valid
 // in NATS KV keys, so the name must not be used in its raw form here.
 func nameKey(accountID, name string) string {
-	return kvutil.AccountKey(accountID, "name."+hex.EncodeToString([]byte(name)))
+	return kvstore.AccountKey(accountID, "name."+hex.EncodeToString([]byte(name)))
 }
 
 // versionKey returns a version-body key: account.lt-<id>.v<n>.
 func versionKey(accountID, ltID string, n int64) string {
-	return kvutil.AccountKey(accountID, ltID+".v"+strconv.FormatInt(n, 10))
+	return kvstore.AccountKey(accountID, ltID+".v"+strconv.FormatInt(n, 10))
 }
 
 // versionPrefix returns the scan prefix for a template's version bodies.
 func versionPrefix(accountID, ltID string) string {
-	return kvutil.AccountKey(accountID, ltID+".v")
+	return kvstore.AccountKey(accountID, ltID+".v")
 }
 
 // --- record load helpers ---
 
 // getHeaderByID reads a header and its KV entry (for CAS) by launch template id.
 func (s *LaunchTemplateServiceImpl) getHeaderByID(ctx context.Context, accountID, ltID string) (*LaunchTemplateHeader, jetstream.KeyValueEntry, error) {
-	entry, err := s.kv.Get(ctx, kvutil.AccountKey(accountID, ltID))
+	entry, err := s.kv.Get(ctx, kvstore.AccountKey(accountID, ltID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
 			return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdNotFound)
@@ -386,7 +386,7 @@ func (s *LaunchTemplateServiceImpl) claimName(ctx context.Context, accountID, na
 	// header means the name is taken; any other read error (transient fault, or a
 	// concurrent in-flight create whose header is not yet written) fails closed so
 	// the name is never stolen from a possibly-live template.
-	_, herr := s.kv.Get(ctx, kvutil.AccountKey(accountID, string(entry.Value())))
+	_, herr := s.kv.Get(ctx, kvstore.AccountKey(accountID, string(entry.Value())))
 	switch {
 	case herr == nil:
 		return errNameInUse()
@@ -404,7 +404,7 @@ func (s *LaunchTemplateServiceImpl) putHeader(ctx context.Context, accountID str
 	if err != nil {
 		return errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.kv.Put(ctx, kvutil.AccountKey(accountID, h.LaunchTemplateId), data); err != nil {
+	if _, err := s.kv.Put(ctx, kvstore.AccountKey(accountID, h.LaunchTemplateId), data); err != nil {
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 	return nil
@@ -510,13 +510,13 @@ func (s *LaunchTemplateServiceImpl) commitVersion(ctx context.Context, accountID
 // re-reading on a conflict. A non-nil check vets each re-read header and
 // stops the commit with its error.
 func (s *LaunchTemplateServiceImpl) commitHeader(ctx context.Context, op, accountID, ltID string, entry jetstream.KeyValueEntry, check func(*LaunchTemplateHeader) error) error {
-	key := kvutil.AccountKey(accountID, ltID)
+	key := kvstore.AccountKey(accountID, ltID)
 	for range maxVersionRetries {
 		_, err := s.kv.Update(ctx, key, entry.Value(), entry.Revision())
 		if err == nil {
 			return nil
 		}
-		if !kvutil.IsRevisionConflict(err) {
+		if !kvstore.IsRevisionConflict(err) {
 			slog.ErrorContext(ctx, op+": header commit failed", "launchTemplateId", ltID, "err", err)
 			return err
 		}
@@ -590,7 +590,7 @@ func (s *LaunchTemplateServiceImpl) ModifyLaunchTemplate(ctx context.Context, in
 // header deleted meanwhile is NotFound for the identifier the request used.
 func (s *LaunchTemplateServiceImpl) setDefaultVersion(ctx context.Context, accountID string, input *ec2.ModifyLaunchTemplateInput, header *LaunchTemplateHeader, entry jetstream.KeyValueEntry, sel string) (*LaunchTemplateHeader, error) {
 	notFound := fmt.Sprintf("The launch template version %s is not found for the specified launch template.", sel)
-	key := kvutil.AccountKey(accountID, header.LaunchTemplateId)
+	key := kvstore.AccountKey(accountID, header.LaunchTemplateId)
 	for range maxVersionRetries {
 		n, err := s.resolveVersionNumber(ctx, accountID, header, sel)
 		if err != nil {
@@ -611,7 +611,7 @@ func (s *LaunchTemplateServiceImpl) setDefaultVersion(ctx context.Context, accou
 		if err == nil {
 			return header, nil
 		}
-		if !kvutil.IsRevisionConflict(err) {
+		if !kvstore.IsRevisionConflict(err) {
 			slog.ErrorContext(ctx, "ModifyLaunchTemplate: header update failed", "launchTemplateId", header.LaunchTemplateId, "err", err)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
@@ -662,7 +662,7 @@ const maxHeaderDeleteAttempts = 3
 // through the public delete; a header changed in between restarts the sweep.
 func (s *LaunchTemplateServiceImpl) deleteTemplate(ctx context.Context, accountID string, header *LaunchTemplateHeader, entry jetstream.KeyValueEntry) error {
 	ltID := header.LaunchTemplateId
-	headerKey := kvutil.AccountKey(accountID, ltID)
+	headerKey := kvstore.AccountKey(accountID, ltID)
 	for attempt := 1; ; attempt++ {
 		if err := s.deleteVersionBodies(ctx, accountID, ltID); err != nil {
 			return err
@@ -824,7 +824,7 @@ func (s *LaunchTemplateServiceImpl) commitVersionDeletes(ctx context.Context, ac
 	}
 	// A template delete that removed the header before the restore has already
 	// swept its bodies, so the restored ones would be left headerless.
-	if _, gerr := s.kv.Get(ctx, kvutil.AccountKey(accountID, ltID)); errors.Is(gerr, jetstream.ErrKeyNotFound) {
+	if _, gerr := s.kv.Get(ctx, kvstore.AccountKey(accountID, ltID)); errors.Is(gerr, jetstream.ErrKeyNotFound) {
 		for n := range deleted {
 			if derr := s.kv.Delete(ctx, versionKey(accountID, ltID, n)); derr != nil {
 				slog.ErrorContext(ctx, "DeleteLaunchTemplateVersions: failed to remove a restored version", "launchTemplateId", ltID, "version", n, "err", derr)

spinifex/domains/ec2/launchtemplate/service_impl_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -116,7 +116,7 @@ func TestCreateLaunchTemplate_OrphanNameReclaim(t *testing.T) {
 	lt := createTemplate(t, svc, "orphan", "t3.micro")
 
 	// Simulate a crash after the name claim but with the header lost.
-	require.NoError(t, svc.kv.Delete(t.Context(), kvutil.AccountKey(testAccountID, aws.StringValue(lt.LaunchTemplateId))))
+	require.NoError(t, svc.kv.Delete(t.Context(), kvstore.AccountKey(testAccountID, aws.StringValue(lt.LaunchTemplateId))))
 
 	// The name index still points at the now-orphaned id; create must reclaim it.
 	out, err := svc.CreateLaunchTemplate(context.Background(), &ec2.CreateLaunchTemplateInput{

spinifex/domains/ec2/launchtemplate/template_lifecycle_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -38,7 +38,7 @@ func TestAccountTermination_VersionCreateRacingADeleteLeavesNoBody(t *testing.T)
 		t.Run(pause, func(t *testing.T) {
 			svc := setupTestService(t)
 			id := createWithUserData(t, svc, "web")
-			header := kvutil.AccountKey(testAccountID, id)
+			header := kvstore.AccountKey(testAccountID, id)
 			raced := false
 			race := func(key string, at string) {
 				if raced || pause != at {
@@ -71,7 +71,7 @@ func TestAccountTermination_DeleteRacingACommittedVersionRemovesIt(t *testing.T)
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeDelete: func(key string) {
 		if key != header || raced {
@@ -95,7 +95,7 @@ func TestAccountTermination_FailedVersionCommitRemovesTheBody(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	before := accountKeys(t, svc.kv, testAccountID)
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	svc.kv = &faultyKV{KeyValue: svc.kv, failUpdate: func(key string) bool { return key == header }}
 
 	_, err := svc.CreateLaunchTemplateVersion(context.Background(), newVersionInput(id), testAccountID)
@@ -110,7 +110,7 @@ func TestAccountTermination_VersionCommitKeepsAConcurrentHeaderChange(t *testing
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header || raced {
@@ -215,10 +215,10 @@ func TestAccountTemplateReferences_FailsClosed(t *testing.T) {
 			require.NoError(t, err)
 		},
 		"header unreadable": func(t *testing.T, svc *LaunchTemplateServiceImpl, id string) {
-			svc.kv = &faultyKV{KeyValue: svc.kv, failGet: func(key string) bool { return key == kvutil.AccountKey(testAccountID, id) }}
+			svc.kv = &faultyKV{KeyValue: svc.kv, failGet: func(key string) bool { return key == kvstore.AccountKey(testAccountID, id) }}
 		},
 		"header undecodable": func(t *testing.T, svc *LaunchTemplateServiceImpl, _ string) {
-			_, err := svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "lt-0broken000000000"), []byte("{not json"))
+			_, err := svc.kv.Put(t.Context(), kvstore.AccountKey(testAccountID, "lt-0broken000000000"), []byte("{not json"))
 			require.NoError(t, err)
 		},
 	}
@@ -241,7 +241,7 @@ func TestModifyLaunchTemplate_RacingADeleteIsNotFound(t *testing.T) {
 		t.Run(map[bool]string{false: "by-id", true: "by-name"}[byName], func(t *testing.T) {
 			svc := setupTestService(t)
 			id := createWithUserData(t, svc, "web")
-			header := kvutil.AccountKey(testAccountID, id)
+			header := kvstore.AccountKey(testAccountID, id)
 			raced := false
 			svc.kv = &faultyKV{KeyValue: svc.kv, beforeUpdate: func(key string) {
 				if key == header && !raced {
@@ -275,7 +275,7 @@ func TestModifyLaunchTemplate_RetriesAConcurrentHeaderChange(t *testing.T) {
 	_, err := svc.CreateLaunchTemplateVersion(context.Background(), newVersionInput(id), testAccountID) // v3
 	require.NoError(t, err)
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header || raced {
@@ -306,7 +306,7 @@ func TestModifyLaunchTemplate_ConflictRetriesAreBounded(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	attempts := 0
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header {
@@ -333,7 +333,7 @@ func TestModifyLaunchTemplate_ConflictReReadFailureIsInternal(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	conflicted := false
 	svc.kv = &faultyKV{
 		KeyValue: inner,
@@ -410,7 +410,7 @@ func TestDefaultChangeRacingAVersionDeleteFailsNotFound(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header || raced {
@@ -460,7 +460,7 @@ func TestFailedVersionDeleteCommitRestoresTheBodies(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	before := accountKeys(t, svc.kv, testAccountID)
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	svc.kv = &faultyKV{KeyValue: svc.kv, failUpdate: func(key string) bool { return key == header }}
 
 	_, err := deleteVersions(svc, id, "2")
@@ -474,7 +474,7 @@ func TestFailedVersionDeleteCommitRestoresTheBodies(t *testing.T) {
 func TestVersionDeleteRacingATemplateDeleteLeavesNothing(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: svc.kv, beforeUpdate: func(key string) {
 		if key != header || raced {
@@ -531,7 +531,7 @@ func TestDefaultChangeRetriesAfterAConcurrentVersionCreate(t *testing.T) {
 	svc := setupTestService(t)
 	id := createWithUserData(t, svc, "web")
 	inner := svc.kv
-	header := kvutil.AccountKey(testAccountID, id)
+	header := kvstore.AccountKey(testAccountID, id)
 	raced := false
 	svc.kv = &faultyKV{KeyValue: inner, beforeUpdate: func(key string) {
 		if key != header || raced {

spinifex/domains/ec2/launchtemplate/template_references.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -27,7 +27,7 @@ type TemplateReferences struct {
 // header or body it cannot read or decode, is an error, because the unread
 // record may name the resource. Headerless bodies resolve from no launch.
 func (s *LaunchTemplateServiceImpl) AccountTemplateReferences(ctx context.Context, accountID string) (*TemplateReferences, error) {
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list launch-template keys: %w", err)
 	}

spinifex/domains/ec2/natgw/lifecycle_invariants_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	ec2eip "github.com/mulgadc/spinifex/spinifex/domains/ec2/eip"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -42,7 +42,7 @@ func TestRLC3_NatGatewayDeletesWhileRouted(t *testing.T) {
 		natgwID := createTestNatGateway(t, svc)
 
 		testutil.SeedKV(t, js, kvBucketRouteTables, map[string][]byte{
-			kvutil.AccountKey(testAccountID, "rtb-routed"): fmt.Appendf(nil,
+			kvstore.AccountKey(testAccountID, "rtb-routed"): fmt.Appendf(nil,
 				`{"route_table_id":"rtb-routed","vpc_id":"vpc-test1","routes":[{"destination_cidr_block":"0.0.0.0/0","nat_gateway_id":%q}]}`, natgwID),
 		})
 
@@ -93,7 +93,7 @@ func createTestNatGateway(t *testing.T, svc *NatGatewayServiceImpl) string {
 
 func readEIP(t *testing.T, svc *NatGatewayServiceImpl, allocID string) ec2eip.EIPRecord {
 	t.Helper()
-	entry, err := svc.eipKV.Get(t.Context(), kvutil.AccountKey(testAccountID, allocID))
+	entry, err := svc.eipKV.Get(t.Context(), kvstore.AccountKey(testAccountID, allocID))
 	require.NoError(t, err)
 	var eip ec2eip.EIPRecord
 	require.NoError(t, json.Unmarshal(entry.Value(), &eip))

spinifex/domains/ec2/natgw/service_impl.go

@@ -20,7 +20,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -45,7 +45,7 @@ func NewNatGatewayServiceImplWithNATS(ctx context.Context, natsConn *nats.Conn)
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	natgwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketNatGateways, 10)
+	natgwKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketNatGateways, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketNatGateways, err)
 	}
@@ -61,17 +61,17 @@ func NewNatGatewayServiceImplWithNATS(ctx context.Context, natsConn *nats.Conn)
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketDeletedNatGateways, err)
 	}
 
-	eipKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
+	eipKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get EIP KV bucket: %w", err)
 	}
 
-	subnetKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketSubnets, 10)
+	subnetKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketSubnets, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get subnet KV bucket: %w", err)
 	}
 
-	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
@@ -92,7 +92,7 @@ func NewNatGatewayServiceImplWithNATS(ctx context.Context, natsConn *nats.Conn)
 // for as long as a client may still ask about it.
 func getOrCreateDeletedBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) {
 	// Terraform polls DescribeNatGateways after delete and expects state=deleted.
-	return kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{
+	return kvstore.GetOrCreateBucketWithOptions(ctx, js, kvstore.BucketOptions{
 		Name:        KVBucketDeletedNatGateways,
 		Description: "Deleted NAT Gateways (auto-expire after 1 hour)",
 		History:     1,
@@ -113,7 +113,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 	allocID := *input.AllocationId
 
 	// Validate subnet exists and get its VPC
-	subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, subnetID))
+	subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, subnetID))
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound)
 	}
@@ -123,7 +123,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 	}
 
 	// Validate EIP exists and is not already associated
-	eipEntry, err := s.eipKV.Get(ctx, kvutil.AccountKey(accountID, allocID))
+	eipEntry, err := s.eipKV.Get(ctx, kvstore.AccountKey(accountID, allocID))
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidAllocationIDNotFound)
 	}
@@ -152,7 +152,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.natgwKV.Put(ctx, kvutil.AccountKey(accountID, natgwID), data); err != nil {
+	if _, err := s.natgwKV.Put(ctx, kvstore.AccountKey(accountID, natgwID), data); err != nil {
 		slog.ErrorContext(ctx, "Failed to store NAT Gateway", "natGatewayId", natgwID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
@@ -165,7 +165,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 	eipRecord.VpcId = subnetRecord.VpcId
 	eipRecord.State = "associated"
 	if eipData, err := json.Marshal(eipRecord); err == nil {
-		if _, err := s.eipKV.Update(ctx, kvutil.AccountKey(accountID, allocID), eipData, eipEntry.Revision()); err != nil {
+		if _, err := s.eipKV.Update(ctx, kvstore.AccountKey(accountID, allocID), eipData, eipEntry.Revision()); err != nil {
 			slog.WarnContext(ctx, "CreateNatGateway: failed to mark EIP associated", "natGatewayId", natgwID, "allocationId", allocID, "err", err)
 		}
 	}
@@ -185,7 +185,7 @@ func (s *NatGatewayServiceImpl) DeleteNatGateway(ctx context.Context, input *ec2
 	}
 
 	natgwID := *input.NatGatewayId
-	key := kvutil.AccountKey(accountID, natgwID)
+	key := kvstore.AccountKey(accountID, natgwID)
 
 	entry, err := s.natgwKV.Get(ctx, key)
 	if err != nil {
@@ -255,7 +255,7 @@ func (s *NatGatewayServiceImpl) publishDeleteEventsForNatGateway(ctx context.Con
 		slog.WarnContext(ctx, "DeleteNatGateway: JetStream setup failed, SNAT teardown deferred to route delete", "natGatewayId", record.NatGatewayId, "err", err)
 		return
 	}
-	rtbKV, err := kvutil.GetOrCreateBucket(ctx, js, kvBucketRouteTables, 10)
+	rtbKV, err := kvstore.GetOrCreateBucket(ctx, js, kvBucketRouteTables, 10)
 	if err != nil {
 		slog.WarnContext(ctx, "DeleteNatGateway: route-table scan failed, SNAT teardown deferred to route delete", "natGatewayId", record.NatGatewayId, "err", err)
 		return
@@ -266,7 +266,7 @@ func (s *NatGatewayServiceImpl) publishDeleteEventsForNatGateway(ctx context.Con
 	}
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := rtbKV.Get(ctx, key)
@@ -302,7 +302,7 @@ func (s *NatGatewayServiceImpl) publishDeleteEventsForNatGateway(ctx context.Con
 			if assoc.SubnetId == "" {
 				continue
 			}
-			subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, assoc.SubnetId))
+			subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, assoc.SubnetId))
 			if err != nil {
 				continue
 			}
@@ -322,7 +322,7 @@ func (s *NatGatewayServiceImpl) disassociateEIP(ctx context.Context, record *Nat
 	if record.AllocationId == "" {
 		return
 	}
-	key := kvutil.AccountKey(accountID, record.AllocationId)
+	key := kvstore.AccountKey(accountID, record.AllocationId)
 	entry, err := s.eipKV.Get(ctx, key)
 	if err != nil {
 		if !errors.Is(err, jetstream.ErrKeyNotFound) {
@@ -380,7 +380,7 @@ func (s *NatGatewayServiceImpl) DescribeNatGateways(ctx context.Context, input *
 	foundIDs := make(map[string]bool)
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 
@@ -415,7 +415,7 @@ func (s *NatGatewayServiceImpl) DescribeNatGateways(ctx context.Context, input *
 		if foundIDs[id] {
 			continue
 		}
-		key := kvutil.AccountKey(accountID, id)
+		key := kvstore.AccountKey(accountID, id)
 		entry, err := s.deletedNatgwKV.Get(ctx, key)
 		if err != nil {
 			if errors.Is(err, jetstream.ErrKeyNotFound) {

spinifex/domains/ec2/natgw/service_impl_test.go

@@ -11,7 +11,7 @@ import (
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -31,18 +31,18 @@ func setupTestServiceJS(t *testing.T) (*NatGatewayServiceImpl, jetstream.JetStre
 
 	// Seed VPC KV
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`),
+		kvstore.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`),
 	})
 
 	// Seed subnet KV (public subnet)
 	testutil.SeedKV(t, js, ec2vpc.KVBucketSubnets, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "subnet-pub1"): []byte(`{"subnet_id":"subnet-pub1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available","map_public_ip_on_launch":true}`),
+		kvstore.AccountKey(testAccountID, "subnet-pub1"): []byte(`{"subnet_id":"subnet-pub1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available","map_public_ip_on_launch":true}`),
 	})
 
 	// Seed EIP KV
 	testutil.SeedKV(t, js, ec2eip.KVBucketEIPs, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "eipalloc-test1"): []byte(`{"allocation_id":"eipalloc-test1","public_ip":"203.0.113.50","state":"allocated"}`),
-		kvutil.AccountKey(testAccountID, "eipalloc-used"):  []byte(`{"allocation_id":"eipalloc-used","public_ip":"203.0.113.51","state":"associated","association_id":"eipassoc-xxx"}`),
+		kvstore.AccountKey(testAccountID, "eipalloc-test1"): []byte(`{"allocation_id":"eipalloc-test1","public_ip":"203.0.113.50","state":"allocated"}`),
+		kvstore.AccountKey(testAccountID, "eipalloc-used"):  []byte(`{"allocation_id":"eipalloc-used","public_ip":"203.0.113.51","state":"associated","association_id":"eipassoc-xxx"}`),
 	})
 
 	svc, err := NewNatGatewayServiceImplWithNATS(t.Context(), nc)
@@ -170,7 +170,7 @@ func TestDeleteNatGateway(t *testing.T) {
 	assert.Equal(t, natgwID, *deleteOut.NatGatewayId)
 
 	// Should be gone
-	_, err = svc.natgwKV.Get(t.Context(), kvutil.AccountKey(testAccountID, natgwID))
+	_, err = svc.natgwKV.Get(t.Context(), kvstore.AccountKey(testAccountID, natgwID))
 	assert.ErrorIs(t, err, jetstream.ErrKeyNotFound)
 }
 

spinifex/domains/ec2/placementgroup/account_cleanup.go

@@ -8,7 +8,7 @@ import (
 	"log/slog"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -39,7 +39,7 @@ type AccountCleanup struct {
 // referenced returns a reason is kept; unreadable records are left for the owner scan to report. A
 // listing failure is returned and nothing is deleted.
 func (s *PlacementGroupServiceImpl) CleanupAccount(ctx context.Context, accountID string, referenced func(groupID, groupName string) string) (*AccountCleanup, error) {
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list placement-group keys: %w", err)
 	}
@@ -47,7 +47,7 @@ func (s *PlacementGroupServiceImpl) CleanupAccount(ctx context.Context, accountI
 	prefix := accountID + "."
 	out := &AccountCleanup{Referenced: map[string]string{}}
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.kv.Get(ctx, key)

spinifex/domains/ec2/placementgroup/account_scan.go

@@ -8,7 +8,7 @@ import (
 	"log/slog"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -34,7 +34,7 @@ type AccountScan struct {
 // A listing failure is returned as an error; per-record read failures are
 // counted. The bucket's schema-version key belongs to no account.
 func (s *PlacementGroupServiceImpl) ScanAccount(ctx context.Context, accountID string) (*AccountScan, error) {
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list placement-group keys: %w", err)
 	}
@@ -42,7 +42,7 @@ func (s *PlacementGroupServiceImpl) ScanAccount(ctx context.Context, accountID s
 	prefix := accountID + "."
 	scan := &AccountScan{}
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.kv.Get(ctx, key)

spinifex/domains/ec2/placementgroup/account_scan_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit/readonlystore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -42,14 +42,14 @@ func TestScanAccountClassifiesPlacementGroupRecords(t *testing.T) {
 		GroupName: "held", NodeInstances: map[string][]string{"node-2": {"i-0bbb", "i-0aaa"}, "node-3": {"i-0aaa"}},
 	}, testAccountID)
 	require.NoError(t, err)
-	_, err = svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "undecodable"), []byte("{not json"))
+	_, err = svc.kv.Put(t.Context(), kvstore.AccountKey(testAccountID, "undecodable"), []byte("{not json"))
 	require.NoError(t, err)
-	_, err = svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "anonymous"), []byte(`{"group_name":"anonymous"}`))
+	_, err = svc.kv.Put(t.Context(), kvstore.AccountKey(testAccountID, "anonymous"), []byte(`{"group_name":"anonymous"}`))
 	require.NoError(t, err)
 	createTestGroup(t, svc, "unreadable", ec2.PlacementStrategySpread)
-	require.Contains(t, bucketKeys(t, svc.kv), kvutil.VersionKey)
+	require.Contains(t, bucketKeys(t, svc.kv), kvstore.VersionKey)
 
-	svc.kv = &hookedKV{KeyValue: readonlystore.NewKV(t, svc.kv), failGet: kvutil.AccountKey(testAccountID, "unreadable")}
+	svc.kv = &hookedKV{KeyValue: readonlystore.NewKV(t, svc.kv), failGet: kvstore.AccountKey(testAccountID, "unreadable")}
 	scan, err := svc.ScanAccount(context.Background(), testAccountID)
 	require.NoError(t, err)
 

spinifex/domains/ec2/placementgroup/account_termination_current_behaviour_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -84,7 +84,7 @@ func TestCurrentBehaviour_AccountTermination_PlacementGroupStorageLayout(t *test
 	}, testAccountID)
 	require.NoError(t, err)
 
-	assert.Equal(t, []string{testAccountID + ".spread-a", kvutil.VersionKey}, bucketKeys(t, svc.kv))
+	assert.Equal(t, []string{testAccountID + ".spread-a", kvstore.VersionKey}, bucketKeys(t, svc.kv))
 
 	entry, err := svc.kv.Get(t.Context(), testAccountID+".spread-a")
 	require.NoError(t, err)
@@ -100,7 +100,7 @@ func TestCurrentBehaviour_AccountTermination_PlacementGroupStorageLayout(t *test
 // a record cannot be removed through DeletePlacementGroup either, and a read
 // failure is an internal error, never an unknown group.
 func TestCurrentBehaviour_AccountTermination_DescribePlacementGroupsHidesUnreadableRecords(t *testing.T) {
-	brokenKey := kvutil.AccountKey(testAccountID, "broken")
+	brokenKey := kvstore.AccountKey(testAccountID, "broken")
 	tests := []struct {
 		name       string
 		seed       func(t *testing.T, svc *PlacementGroupServiceImpl)
@@ -139,7 +139,7 @@ func TestCurrentBehaviour_AccountTermination_DescribePlacementGroupsHidesUnreada
 func TestAccountTermination_DeletePlacementGroupLosesToAConcurrentMember(t *testing.T) {
 	svc := setupTestService(t)
 	createTestGroup(t, svc, "cluster-a", ec2.PlacementStrategyCluster)
-	key := kvutil.AccountKey(testAccountID, "cluster-a")
+	key := kvstore.AccountKey(testAccountID, "cluster-a")
 
 	launcher := &PlacementGroupServiceImpl{kv: svc.kv, region: svc.region}
 	var reserveErr, finalizeErr error
@@ -222,7 +222,7 @@ func TestAccountCleanup_DeletesEmptyGroupsAndBlocksOnMembers(t *testing.T) {
 	empty := createTestGroup(t, svc, "empty", ec2.PlacementStrategySpread)
 	held := createTestGroup(t, svc, "held", ec2.PlacementStrategyCluster)
 	require.NoError(t, addMember(t, svc, "held", "i-0123456789abcdef0"))
-	_, err := svc.kv.Put(t.Context(), kvutil.AccountKey(testAccountID, "broken"), []byte("{not json"))
+	_, err := svc.kv.Put(t.Context(), kvstore.AccountKey(testAccountID, "broken"), []byte("{not json"))
 	require.NoError(t, err)
 	other := "999988887777"
 	_, err = svc.CreatePlacementGroup(context.Background(), &ec2.CreatePlacementGroupInput{
@@ -238,7 +238,7 @@ func TestAccountCleanup_DeletesEmptyGroupsAndBlocksOnMembers(t *testing.T) {
 
 	assert.Equal(t, []string{"held"}, describedGroupNames(t, svc, testAccountID))
 	assert.Equal(t, []string{"theirs"}, describedGroupNames(t, svc, other))
-	_, err = svc.kv.Get(t.Context(), kvutil.AccountKey(testAccountID, "broken"))
+	_, err = svc.kv.Get(t.Context(), kvstore.AccountKey(testAccountID, "broken"))
 	require.NoError(t, err, "an unreadable record is never deleted")
 }
 
@@ -247,7 +247,7 @@ func TestAccountCleanup_DeletesEmptyGroupsAndBlocksOnMembers(t *testing.T) {
 func TestAccountCleanup_ConcurrentMemberBlocksTheDelete(t *testing.T) {
 	svc := setupTestService(t)
 	group := createTestGroup(t, svc, "cluster-a", ec2.PlacementStrategyCluster)
-	key := kvutil.AccountKey(testAccountID, "cluster-a")
+	key := kvstore.AccountKey(testAccountID, "cluster-a")
 	launcher := &PlacementGroupServiceImpl{kv: svc.kv, region: svc.region}
 	var memberErr error
 	// Read 1 is the cleanup's listing read; read 2 is the delete's check.

spinifex/domains/ec2/placementgroup/service_impl.go

@@ -17,7 +17,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -87,7 +87,7 @@ func NewPlacementGroupServiceImplWithNATS(ctx context.Context, cfg *config.Confi
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketPlacementGroups, 10)
+	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketPlacementGroups, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketPlacementGroups, err)
 	}
@@ -137,7 +137,7 @@ func (s *PlacementGroupServiceImpl) CreatePlacementGroup(ctx context.Context, in
 	}
 
 	groupName := *input.GroupName
-	key := kvutil.AccountKey(accountID, groupName)
+	key := kvstore.AccountKey(accountID, groupName)
 	groupID := awsidentifiers.GenerateResourceID("pg")
 
 	record := PlacementGroupRecord{
@@ -175,7 +175,7 @@ func (s *PlacementGroupServiceImpl) DeletePlacementGroup(ctx context.Context, in
 	}
 
 	groupName := *input.GroupName
-	if _, err := s.deleteGroup(ctx, accountID, kvutil.AccountKey(accountID, groupName)); err != nil {
+	if _, err := s.deleteGroup(ctx, accountID, kvstore.AccountKey(accountID, groupName)); err != nil {
 		if errors.Is(err, errPGAbsent) {
 			// Lower case, unlike DescribePlacementGroups' message for the same code.
 			return nil, awserrors.Errorf(awserrors.ErrorInvalidPlacementGroupUnknown, "The placement group '%s' is unknown.", groupName)
@@ -194,7 +194,7 @@ func (s *PlacementGroupServiceImpl) DeletePlacementGroup(ctx context.Context, in
 // The returned count is the members seen by the last check.
 func (s *PlacementGroupServiceImpl) deleteGroup(ctx context.Context, accountID, key string) (int, error) {
 	members := 0
-	record, err := kvutil.DeleteIf(ctx, s.kv, key, casConfig(), func(record *PlacementGroupRecord) error {
+	record, err := kvstore.DeleteIf(ctx, s.kv, key, casConfig(), func(record *PlacementGroupRecord) error {
 		members = memberCount(record.NodeInstances)
 		if members > 0 {
 			return errPGInUse
@@ -238,7 +238,7 @@ func (s *PlacementGroupServiceImpl) DescribePlacementGroups(ctx context.Context,
 
 	var groups []*ec2.PlacementGroup
 	for _, k := range keys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -341,7 +341,7 @@ func pgMatchesFilters(record *PlacementGroupRecord, filters map[string][]string)
 
 const maxCASRetries = 5
 
-// errPGAbsent and errPGContended mark the two kvutil outcomes that map to
+// errPGAbsent and errPGContended mark the two kvstore outcomes that map to
 // specific AWS errors rather than a generic internal failure, and neither
 // escapes this file. errPGInUse is the AWS error itself.
 var (
@@ -350,8 +350,8 @@ var (
 	errPGInUse     = errors.New(awserrors.ErrorInvalidPlacementGroupInUse)
 )
 
-// mapCASError turns a kvutil failure into the AWS error the caller returns.
-// A non-kvutil error came from mutate and is already an AWS code.
+// mapCASError turns a kvstore failure into the AWS error the caller returns.
+// A non-kvstore error came from mutate and is already an AWS code.
 func mapCASError(ctx context.Context, op, groupName, accountID string, err error) error {
 	switch {
 	case errors.Is(err, errPGAbsent):
@@ -359,16 +359,16 @@ func mapCASError(ctx context.Context, op, groupName, accountID string, err error
 	case errors.Is(err, errPGContended):
 		slog.ErrorContext(ctx, "Placement group CAS retries exhausted under contention",
 			"op", op, "groupName", groupName, "accountID", accountID)
-	case errors.Is(err, kvutil.ErrRead):
+	case errors.Is(err, kvstore.ErrRead):
 		slog.ErrorContext(ctx, "Failed to read placement group from KV",
 			"op", op, "groupName", groupName, "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrDecode):
+	case errors.Is(err, kvstore.ErrDecode):
 		slog.ErrorContext(ctx, "Corrupt placement group record in KV",
 			"op", op, "groupName", groupName, "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrEncode):
+	case errors.Is(err, kvstore.ErrEncode):
 		slog.ErrorContext(ctx, "Failed to marshal placement group record",
 			"op", op, "groupName", groupName, "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrWrite):
+	case errors.Is(err, kvstore.ErrWrite):
 		slog.ErrorContext(ctx, "Failed to write placement group to KV",
 			"op", op, "groupName", groupName, "accountID", accountID, "err", err)
 	default:
@@ -378,8 +378,8 @@ func mapCASError(ctx context.Context, op, groupName, accountID string, err error
 }
 
 // casConfig is the shared CAS policy for every placement group mutation.
-func casConfig() kvutil.CASConfig {
-	return kvutil.CASConfig{
+func casConfig() kvstore.CASConfig {
+	return kvstore.CASConfig{
 		Attempts:  maxCASRetries,
 		NotFound:  errPGAbsent,
 		Exhausted: func(string, int) error { return errPGContended },
@@ -394,7 +394,7 @@ func (s *PlacementGroupServiceImpl) ReserveSpreadNodes(ctx context.Context, inpu
 	}
 
 	var selected []string
-	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			if record.State != ec2.PlacementGroupStateAvailable {
 				return false, errors.New(awserrors.ErrorInvalidPlacementGroupUnknown)
@@ -437,7 +437,7 @@ func (s *PlacementGroupServiceImpl) FinalizeSpreadInstances(ctx context.Context,
 		return nil, errors.New(awserrors.ErrorMissingParameter)
 	}
 
-	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			maps.Copy(record.NodeInstances, input.NodeInstances)
 			return true, nil
@@ -462,7 +462,7 @@ func (s *PlacementGroupServiceImpl) ReleaseSpreadNodes(ctx context.Context, inpu
 		releaseSet[n] = true
 	}
 
-	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			changed := false
 			for node := range releaseSet {
@@ -490,7 +490,7 @@ func (s *PlacementGroupServiceImpl) RemoveInstance(ctx context.Context, input *R
 		return nil, errors.New(awserrors.ErrorMissingParameter)
 	}
 
-	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			instances, exists := record.NodeInstances[input.NodeName]
 			if !exists {
@@ -533,7 +533,7 @@ func (s *PlacementGroupServiceImpl) ReserveClusterNode(ctx context.Context, inpu
 	}
 
 	var targetNode string
-	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			if record.State != ec2.PlacementGroupStateAvailable {
 				return false, errors.New(awserrors.ErrorInvalidPlacementGroupUnknown)
@@ -570,7 +570,7 @@ func (s *PlacementGroupServiceImpl) FinalizeClusterInstances(ctx context.Context
 		return nil, errors.New(awserrors.ErrorMissingParameter)
 	}
 
-	_, err := kvutil.Update(ctx, s.kv, kvutil.AccountKey(accountID, input.GroupName), casConfig(),
+	_, err := kvstore.Update(ctx, s.kv, kvstore.AccountKey(accountID, input.GroupName), casConfig(),
 		func(record *PlacementGroupRecord) (bool, error) {
 			// Append rather than replace: a cluster group can have several launches in flight at once.
 			for node, ids := range input.NodeInstances {

spinifex/domains/ec2/placementgroup/service_impl_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -39,7 +39,7 @@ func createTestGroup(t *testing.T, svc *PlacementGroupServiceImpl, name, strateg
 // readRecord returns the stored record for assertions.
 func readRecord(t *testing.T, svc *PlacementGroupServiceImpl, groupName string) *PlacementGroupRecord {
 	t.Helper()
-	entry, err := svc.kv.Get(t.Context(), kvutil.AccountKey(testAccountID, groupName))
+	entry, err := svc.kv.Get(t.Context(), kvstore.AccountKey(testAccountID, groupName))
 	require.NoError(t, err)
 	var record PlacementGroupRecord
 	require.NoError(t, json.Unmarshal(entry.Value(), &record))
@@ -50,7 +50,7 @@ func readRecord(t *testing.T, svc *PlacementGroupServiceImpl, groupName string)
 // starting state the public API cannot produce.
 func seedRecord(t *testing.T, svc *PlacementGroupServiceImpl, groupName string, mutate func(*PlacementGroupRecord)) {
 	t.Helper()
-	_, err := kvutil.Update(t.Context(), svc.kv, kvutil.AccountKey(testAccountID, groupName), kvutil.CASConfig{},
+	_, err := kvstore.Update(t.Context(), svc.kv, kvstore.AccountKey(testAccountID, groupName), kvstore.CASConfig{},
 		func(record *PlacementGroupRecord) (bool, error) {
 			mutate(record)
 			return true, nil

spinifex/domains/ec2/routetable/service_impl.go

@@ -25,7 +25,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -52,7 +52,7 @@ func NewRouteTableServiceImplWithNATS(ctx context.Context, cfg *config.Config, n
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	rtbKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketRouteTables, 10)
+	rtbKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketRouteTables, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketRouteTables, err)
 	}
@@ -60,22 +60,22 @@ func NewRouteTableServiceImplWithNATS(ctx context.Context, cfg *config.Config, n
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketRouteTables, err)
 	}
 
-	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
 
-	igwKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2igw.KVBucketIGW, 10)
+	igwKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2igw.KVBucketIGW, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get IGW KV bucket: %w", err)
 	}
 
-	subnetKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketSubnets, 10)
+	subnetKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketSubnets, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get subnet KV bucket: %w", err)
 	}
 
-	natgwKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2natgw.KVBucketNatGateways, 10)
+	natgwKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2natgw.KVBucketNatGateways, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get NAT Gateway KV bucket: %w", err)
 	}
@@ -95,7 +95,7 @@ func NewRouteTableServiceImplWithNATS(ctx context.Context, cfg *config.Config, n
 
 // getRouteTable retrieves a route table record from KV.
 func (s *RouteTableServiceImpl) getRouteTable(ctx context.Context, accountID, rtbID string) (*RouteTableRecord, error) {
-	entry, err := s.rtbKV.Get(ctx, kvutil.AccountKey(accountID, rtbID))
+	entry, err := s.rtbKV.Get(ctx, kvstore.AccountKey(accountID, rtbID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
 			return nil, rtbNotFoundError(rtbID)
@@ -118,7 +118,7 @@ func (s *RouteTableServiceImpl) putRouteTable(ctx context.Context, accountID str
 		slog.ErrorContext(ctx, "Failed to marshal route table record", "routeTableId", record.RouteTableId, "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.rtbKV.Put(ctx, kvutil.AccountKey(accountID, record.RouteTableId), data); err != nil {
+	if _, err := s.rtbKV.Put(ctx, kvstore.AccountKey(accountID, record.RouteTableId), data); err != nil {
 		slog.ErrorContext(ctx, "Failed to write route table to KV", "routeTableId", record.RouteTableId, "err", err)
 		return errors.New(awserrors.ErrorServerInternal)
 	}
@@ -130,7 +130,7 @@ func (s *RouteTableServiceImpl) putRouteTable(ctx context.Context, accountID str
 // parallel AssociateRouteTable calls Terraform fans out per route table.
 const rtbCASMaxRetries = 16
 
-// errRTBAbsent and errRTBContended mark the two kvutil outcomes that map to
+// errRTBAbsent and errRTBContended mark the two kvstore outcomes that map to
 // specific AWS errors rather than a generic internal failure. Neither
 // escapes this file.
 var (
@@ -143,7 +143,7 @@ var (
 // several subnets with one route table at once. mutate reports whether
 // it changed the record; a false return commits nothing.
 func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, accountID, rtbID string, mutate func(*RouteTableRecord) (bool, error)) error {
-	_, err := kvutil.Update(ctx, s.rtbKV, kvutil.AccountKey(accountID, rtbID), kvutil.CASConfig{
+	_, err := kvstore.Update(ctx, s.rtbKV, kvstore.AccountKey(accountID, rtbID), kvstore.CASConfig{
 		Attempts:  rtbCASMaxRetries,
 		NotFound:  errRTBAbsent,
 		Exhausted: func(string, int) error { return errRTBContended },
@@ -157,13 +157,13 @@ func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, account
 		// Contended rather than broken.
 		slog.ErrorContext(ctx, "Route table CAS retries exhausted under contention",
 			"routeTableId", rtbID, "attempts", rtbCASMaxRetries)
-	case errors.Is(err, kvutil.ErrRead):
+	case errors.Is(err, kvstore.ErrRead):
 		slog.ErrorContext(ctx, "Failed to read route table from KV", "routeTableId", rtbID, "err", err)
-	case errors.Is(err, kvutil.ErrDecode):
+	case errors.Is(err, kvstore.ErrDecode):
 		slog.ErrorContext(ctx, "Corrupt route table record in KV", "routeTableId", rtbID, "err", err)
-	case errors.Is(err, kvutil.ErrEncode):
+	case errors.Is(err, kvstore.ErrEncode):
 		slog.ErrorContext(ctx, "Failed to marshal route table record", "routeTableId", rtbID, "err", err)
-	case errors.Is(err, kvutil.ErrWrite):
+	case errors.Is(err, kvstore.ErrWrite):
 		slog.ErrorContext(ctx, "Failed to write route table to KV", "routeTableId", rtbID, "err", err)
 	default:
 		return err
@@ -173,7 +173,7 @@ func (s *RouteTableServiceImpl) mutateRouteTableCAS(ctx context.Context, account
 
 // getVPCCidr looks up a VPC's CIDR block from the VPC KV bucket.
 func (s *RouteTableServiceImpl) getVPCCidr(ctx context.Context, accountID, vpcID string) (string, error) {
-	entry, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID))
+	entry, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID))
 	if err != nil {
 		return "", awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
 	}
@@ -229,7 +229,7 @@ func (s *RouteTableServiceImpl) subnetsImplicitlyOnMainRT(ctx context.Context, a
 	}
 	var implicit []string
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.subnetKV.Get(ctx, key)
@@ -387,7 +387,7 @@ func (s *RouteTableServiceImpl) allSubnetsForVPC(ctx context.Context, accountID,
 	}
 	var subnets []string
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.subnetKV.Get(ctx, key)
@@ -455,7 +455,7 @@ func (s *RouteTableServiceImpl) allRouteTablesForVPC(ctx context.Context, accoun
 
 	var results []RouteTableRecord
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.rtbKV.Get(ctx, key)
@@ -577,7 +577,7 @@ func (s *RouteTableServiceImpl) DeleteRouteTable(ctx context.Context, input *ec2
 		}
 	}
 
-	if err := s.rtbKV.Delete(ctx, kvutil.AccountKey(accountID, rtbID)); err != nil {
+	if err := s.rtbKV.Delete(ctx, kvstore.AccountKey(accountID, rtbID)); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -633,7 +633,7 @@ func (s *RouteTableServiceImpl) DescribeRouteTables(ctx context.Context, input *
 	foundIDs := make(map[string]bool)
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 
@@ -740,7 +740,7 @@ func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.Crea
 	case input.GatewayId != nil && *input.GatewayId != "":
 		igwID := *input.GatewayId
 		// Verify IGW exists and is attached to the same VPC
-		igwEntry, err := s.igwKV.Get(ctx, kvutil.AccountKey(accountID, igwID))
+		igwEntry, err := s.igwKV.Get(ctx, kvstore.AccountKey(accountID, igwID))
 		if err != nil {
 			return nil, gatewayNotFoundError(igwID)
 		}
@@ -770,7 +770,7 @@ func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.Crea
 	case input.NatGatewayId != nil && *input.NatGatewayId != "":
 		natgwID := *input.NatGatewayId
 		// Verify NAT GW exists and belongs to the same VPC
-		natgwEntry, err := s.natgwKV.Get(ctx, kvutil.AccountKey(accountID, natgwID))
+		natgwEntry, err := s.natgwKV.Get(ctx, kvstore.AccountKey(accountID, natgwID))
 		if err != nil {
 			return nil, errors.New(awserrors.ErrorInvalidNatGatewayIDNotFound)
 		}
@@ -924,7 +924,7 @@ func (s *RouteTableServiceImpl) ReplaceRoute(ctx context.Context, input *ec2.Rep
 	igwID := *input.GatewayId
 
 	// AWS reports a missing gateway ahead of a missing route.
-	igwEntry, err := s.igwKV.Get(ctx, kvutil.AccountKey(accountID, igwID))
+	igwEntry, err := s.igwKV.Get(ctx, kvstore.AccountKey(accountID, igwID))
 	if err != nil {
 		return nil, gatewayNotFoundError(igwID)
 	}
@@ -983,7 +983,7 @@ func (s *RouteTableServiceImpl) AssociateRouteTable(ctx context.Context, input *
 	}
 
 	// Verify subnet exists and belongs to the same VPC
-	subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, subnetID))
+	subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, subnetID))
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound)
 	}
@@ -1077,7 +1077,7 @@ func (s *RouteTableServiceImpl) DisassociateRouteTable(ctx context.Context, inpu
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 
@@ -1168,7 +1168,7 @@ func (s *RouteTableServiceImpl) ReplaceRouteTableAssociation(ctx context.Context
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 
@@ -1369,7 +1369,7 @@ func (s *RouteTableServiceImpl) publishNatGatewayDeleteEvents(ctx context.Contex
 	if s.natsConn == nil {
 		return
 	}
-	natgwEntry, err := s.natgwKV.Get(ctx, kvutil.AccountKey(accountID, natgwID))
+	natgwEntry, err := s.natgwKV.Get(ctx, kvstore.AccountKey(accountID, natgwID))
 	if err != nil {
 		slog.WarnContext(ctx, "NAT GW event: natgw lookup failed", "topic", networkv1.NATGatewayDeleteSubject, "natGatewayId", natgwID, "err", err)
 		return
@@ -1417,7 +1417,7 @@ func (s *RouteTableServiceImpl) publishNatGatewayEventsForAssociation(ctx contex
 		if r.NatGatewayId == "" {
 			continue
 		}
-		natgwEntry, err := s.natgwKV.Get(ctx, kvutil.AccountKey(accountID, r.NatGatewayId))
+		natgwEntry, err := s.natgwKV.Get(ctx, kvstore.AccountKey(accountID, r.NatGatewayId))
 		if err != nil {
 			slog.WarnContext(ctx, "NAT GW event: natgw lookup failed", "topic", topic, "natGatewayId", r.NatGatewayId, "err", err)
 			continue
@@ -1442,7 +1442,7 @@ func (s *RouteTableServiceImpl) publishNatGatewayEventForSubnet(ctx context.Cont
 	if s.natsConn == nil {
 		return
 	}
-	subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, subnetID))
+	subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, subnetID))
 	if err != nil {
 		slog.WarnContext(ctx, "NAT GW event: subnet lookup failed", "topic", topic, "subnetId", subnetID, "err", err)
 		return

spinifex/domains/ec2/routetable/service_impl_test.go

@@ -13,7 +13,7 @@ import (
 	ec2natgw "github.com/mulgadc/spinifex/spinifex/domains/ec2/natgw"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -28,23 +28,23 @@ func setupTestService(t *testing.T) *RouteTableServiceImpl {
 
 	// Seed VPC KV
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`),
+		kvstore.AccountKey(testAccountID, "vpc-test1"): []byte(`{"vpc_id":"vpc-test1","cidr_block":"10.0.0.0/16","state":"available"}`),
 	})
 
 	// Seed IGW KV (attached to vpc-test1)
 	testutil.SeedKV(t, js, ec2igw.KVBucketIGW, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "igw-test1"): []byte(`{"internet_gateway_id":"igw-test1","vpc_id":"vpc-test1","state":"available"}`),
+		kvstore.AccountKey(testAccountID, "igw-test1"): []byte(`{"internet_gateway_id":"igw-test1","vpc_id":"vpc-test1","state":"available"}`),
 	})
 
 	// Seed subnet KV
 	testutil.SeedKV(t, js, ec2vpc.KVBucketSubnets, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "subnet-test1"): []byte(`{"subnet_id":"subnet-test1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available"}`),
-		kvutil.AccountKey(testAccountID, "subnet-priv1"): []byte(`{"subnet_id":"subnet-priv1","vpc_id":"vpc-test1","cidr_block":"10.0.11.0/24","state":"available"}`),
+		kvstore.AccountKey(testAccountID, "subnet-test1"): []byte(`{"subnet_id":"subnet-test1","vpc_id":"vpc-test1","cidr_block":"10.0.1.0/24","state":"available"}`),
+		kvstore.AccountKey(testAccountID, "subnet-priv1"): []byte(`{"subnet_id":"subnet-priv1","vpc_id":"vpc-test1","cidr_block":"10.0.11.0/24","state":"available"}`),
 	})
 
 	// Seed NAT Gateway KV
 	testutil.SeedKV(t, js, ec2natgw.KVBucketNatGateways, map[string][]byte{
-		kvutil.AccountKey(testAccountID, "nat-test1"): []byte(`{"nat_gateway_id":"nat-test1","vpc_id":"vpc-test1","subnet_id":"subnet-test1","public_ip":"192.168.1.100","state":"available"}`),
+		kvstore.AccountKey(testAccountID, "nat-test1"): []byte(`{"nat_gateway_id":"nat-test1","vpc_id":"vpc-test1","subnet_id":"subnet-test1","public_ip":"192.168.1.100","state":"available"}`),
 	})
 
 	svc, err := NewRouteTableServiceImplWithNATS(t.Context(), nil, nc)

spinifex/domains/ec2/spotinstance/account_scan.go

@@ -10,7 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/nats-io/nats.go/jetstream"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 )
 
 // AccountSpotRequest is one persisted request in an account scan.
@@ -54,7 +54,7 @@ func scanBucket(ctx context.Context, kv jetstream.KeyValue, accountID string) ([
 	prefix := accountID + "."
 	var out []AccountSpotRequest
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := kv.Get(ctx, key)

spinifex/domains/ec2/spotinstance/account_scan_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 )
 
 func TestScanAccountListsActiveAndTerminalForOneAccount(t *testing.T) {
@@ -35,7 +35,7 @@ func TestScanAccountListsActiveAndTerminalForOneAccount(t *testing.T) {
 func TestScanAccountFailsOnAnUndecodableRecord(t *testing.T) {
 	svc := setupTestService(t)
 	putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"))
-	_, err := svc.activeKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-bad"), []byte("{not json"))
+	_, err := svc.activeKV.Put(t.Context(), kvstore.AccountKey(testAccountID, "sir-bad"), []byte("{not json"))
 	require.NoError(t, err)
 
 	scan, err := svc.ScanAccount(t.Context(), testAccountID)
@@ -45,7 +45,7 @@ func TestScanAccountFailsOnAnUndecodableRecord(t *testing.T) {
 
 func TestScanAccountFailsOnARecordWithoutARequest(t *testing.T) {
 	svc := setupTestService(t)
-	_, err := svc.terminalKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-empty"), []byte(`{"account_id":"x"}`))
+	_, err := svc.terminalKV.Put(t.Context(), kvstore.AccountKey(testAccountID, "sir-empty"), []byte(`{"account_id":"x"}`))
 	require.NoError(t, err)
 
 	_, err = svc.ScanAccount(t.Context(), testAccountID)
@@ -76,7 +76,7 @@ func (k failingGetKV) Get(ctx context.Context, key string) (jetstream.KeyValueEn
 func TestScanAccountFailsWhenARecordCannotBeRead(t *testing.T) {
 	svc := setupTestService(t)
 	putSIRs(t, svc, makeSIR("sir-aaa", "i-aaa"), makeSIR("sir-bbb", "i-bbb"))
-	svc.activeKV = failingGetKV{KeyValue: svc.activeKV, key: kvutil.AccountKey(testAccountID, "sir-bbb")}
+	svc.activeKV = failingGetKV{KeyValue: svc.activeKV, key: kvstore.AccountKey(testAccountID, "sir-bbb")}
 
 	scan, err := svc.ScanAccount(t.Context(), testAccountID)
 	require.Error(t, err)

spinifex/domains/ec2/spotinstance/launch_references.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -24,14 +24,14 @@ type LaunchReferences struct {
 // account. It is complete or an error: an unlistable bucket or an unreadable
 // request is an error, because the unread request may name the resource.
 func (s *SpotInstanceServiceImpl) AccountLaunchReferences(ctx context.Context, accountID string) (*LaunchReferences, error) {
-	keys, err := kvutil.Keys(ctx, s.activeKV)
+	keys, err := kvstore.Keys(ctx, s.activeKV)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list spot requests: %w", err)
 	}
 	prefix := accountID + "."
 	out := &LaunchReferences{}
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		record, err := s.readRecord(ctx, s.activeKV, key)

spinifex/domains/ec2/spotinstance/launch_references_test.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -41,7 +41,7 @@ func TestAccountLaunchReferences_ReadsOpenAndActiveRequests(t *testing.T) {
 func TestAccountLaunchReferences_UnreadableRequestFailsTheCheck(t *testing.T) {
 	svc := setupTestService(t)
 	putSIRs(t, svc, makeSIR("sir-active01", "i-0000000000000001"))
-	_, err := svc.activeKV.Put(t.Context(), kvutil.AccountKey(testAccountID, "sir-broken01"), []byte("{not json"))
+	_, err := svc.activeKV.Put(t.Context(), kvstore.AccountKey(testAccountID, "sir-broken01"), []byte("{not json"))
 	require.NoError(t, err)
 
 	_, err = svc.AccountLaunchReferences(t.Context(), testAccountID)

spinifex/domains/ec2/spotinstance/service_impl.go

@@ -15,7 +15,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -41,7 +41,7 @@ func NewSpotInstanceServiceImplWithNATS(ctx context.Context, cfg *config.Config,
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	activeKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketSpotRequests, 10)
+	activeKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketSpotRequests, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSpotRequests, err)
 	}
@@ -71,7 +71,7 @@ func NewSpotInstanceServiceImplWithNATS(ctx context.Context, cfg *config.Config,
 // getOrCreateTerminalBucket opens the terminal bucket without changing its
 // existing replica or placement configuration, creating it only when absent.
 func getOrCreateTerminalBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) {
-	return kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{
+	return kvstore.GetOrCreateBucketWithOptions(ctx, js, kvstore.BucketOptions{
 		Name:        KVBucketSpotRequestsTerminal,
 		Description: "Terminal Spot Instance Requests (auto-expire after 1 hour)",
 		History:     1,
@@ -91,7 +91,7 @@ func (s *SpotInstanceServiceImpl) PutSpotInstanceRequests(ctx context.Context, i
 		if err != nil {
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
-		if _, err := s.activeKV.Put(ctx, kvutil.AccountKey(accountID, sirID), data); err != nil {
+		if _, err := s.activeKV.Put(ctx, kvstore.AccountKey(accountID, sirID), data); err != nil {
 			slog.ErrorContext(ctx, "PutSpotInstanceRequests: KV put failed", "sirId", sirID, "err", err)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
@@ -171,7 +171,7 @@ func (s *SpotInstanceServiceImpl) CancelSpotInstanceRequests(ctx context.Context
 	cancelled := make([]*ec2.CancelledSpotInstanceRequest, 0, len(input.SpotInstanceRequestIds))
 	for _, idPtr := range input.SpotInstanceRequestIds {
 		sirID := aws.StringValue(idPtr)
-		key := kvutil.AccountKey(accountID, sirID)
+		key := kvstore.AccountKey(accountID, sirID)
 
 		record, err := s.readRecord(ctx, s.activeKV, key)
 		if err == nil {
@@ -213,7 +213,7 @@ func (s *SpotInstanceServiceImpl) CloseForInstance(ctx context.Context, instance
 
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		record, err := s.readRecord(ctx, s.activeKV, key)
@@ -251,7 +251,7 @@ func (s *SpotInstanceServiceImpl) listRequests(ctx context.Context, kv jetstream
 	prefix := accountID + "."
 	var requests []*ec2.SpotInstanceRequest
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		record, err := s.readRecord(ctx, kv, key)

spinifex/domains/ec2/systeminstance/launcher.go

@@ -10,6 +10,42 @@ import "errors"
 // Callers treat it as idempotent success so a retried delete does not block on a gone VM.
 var ErrSystemInstanceNotFound = errors.New("sysinstance: instance not found")
 
+// ErrSystemInstanceNotSystem is returned by TerminateSystemInstance when the ID
+// names a customer instance. Nothing was touched, and it is not "gone": callers
+// must stop and never act on that ID.
+var ErrSystemInstanceNotSystem = errors.New("sysinstance: not a system instance")
+
+// ErrSystemInstanceOwnershipUnknown is returned by TerminateSystemInstance when
+// the instance's record could not be read. Nothing was terminated; callers must
+// stop and retry, never treat it as the VM being gone.
+var ErrSystemInstanceOwnershipUnknown = errors.New("sysinstance: instance ownership unknown")
+
+// Ownership says who an instance ID belongs to.
+type Ownership string
+
+const (
+	// OwnershipSystem is a platform VM: a known system manager in the system account.
+	OwnershipSystem Ownership = "system"
+	// OwnershipCustomer is any other instance that has a record.
+	OwnershipCustomer Ownership = "customer"
+	// OwnershipAbsent is an ID with no live, stopped or terminated record.
+	OwnershipAbsent Ownership = "absent"
+)
+
+// Classification is what the system terminate path learns about an ID. For a
+// system VM it also carries the public IP allocation and ENI recorded at launch.
+type Classification struct {
+	Ownership       Ownership
+	PublicIPAllocID string
+	ENIID           string
+}
+
+// OwnershipClassifier tells the system terminate path whether an ID is a
+// system VM, without exposing instance records to it.
+type OwnershipClassifier interface {
+	Classify(instanceID string) (Classification, error)
+}
+
 // BootMode selects how a system instance boots.
 type BootMode string
 

spinifex/domains/ec2/tagmirror/tagging_current_behaviour_test.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -39,7 +39,7 @@ func (k *interleavingKV) Get(ctx context.Context, key string) (jetstream.KeyValu
 // its write both survive alongside its own tag.
 func TestCurrentBehaviour_Tagging_RecordMirrorKeepsInterleavedWrites(t *testing.T) {
 	const acct, id = "123456789012", "sg-0123456789abcdef0"
-	key := kvutil.AccountKey(acct, id)
+	key := kvstore.AccountKey(acct, id)
 	seed, err := json.Marshal(tagMirrorRecord{Name: "original", Tags: map[string]string{}})
 	require.NoError(t, err)
 	kv := seedKV(t, "tagmirror-cb", map[string][]byte{key: seed})

spinifex/domains/ec2/tagmirror/tagmirror.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -83,8 +83,8 @@ var errRecordAbsent = errors.New("tagmirror: record absent")
 // applying mut and committing at the read revision, retrying on conflict so a
 // concurrent write is never reverted. An absent resource is skipped.
 func UpdateKVRecordTags[R any](ctx context.Context, kv jetstream.KeyValue, accountID, resourceID string, mut func(*R)) error {
-	key := kvutil.AccountKey(accountID, resourceID)
-	_, err := kvutil.Update(ctx, kv, key, kvutil.CASConfig{NotFound: errRecordAbsent}, func(rec *R) (bool, error) {
+	key := kvstore.AccountKey(accountID, resourceID)
+	_, err := kvstore.Update(ctx, kv, key, kvstore.CASConfig{NotFound: errRecordAbsent}, func(rec *R) (bool, error) {
 		mut(rec)
 		return true, nil
 	})

spinifex/domains/ec2/tagmirror/tagmirror_test.go

@@ -8,7 +8,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -75,15 +75,15 @@ func TestUpdateKVRecordTags(t *testing.T) {
 	rec, err := json.Marshal(&tagMirrorRecord{Name: "r1", Tags: map[string]string{"a": "1"}})
 	require.NoError(t, err)
 	kv := seedKV(t, "tagmirror-test", map[string][]byte{
-		kvutil.AccountKey("acct", "res-1"):   rec,
-		kvutil.AccountKey("acct", "res-bad"): []byte("not-json"),
+		kvstore.AccountKey("acct", "res-1"):   rec,
+		kvstore.AccountKey("acct", "res-bad"): []byte("not-json"),
 	})
 
 	t.Run("mutates and persists", func(t *testing.T) {
 		require.NoError(t, UpdateKVRecordTags(t.Context(), kv, "acct", "res-1", func(r *tagMirrorRecord) {
 			r.Tags["b"] = "2"
 		}))
-		entry, err := kv.Get(t.Context(), kvutil.AccountKey("acct", "res-1"))
+		entry, err := kv.Get(t.Context(), kvstore.AccountKey("acct", "res-1"))
 		require.NoError(t, err)
 		var got tagMirrorRecord
 		require.NoError(t, json.Unmarshal(entry.Value(), &got))
@@ -110,14 +110,14 @@ func TestMirrorKVRecordTags(t *testing.T) {
 	nilTags, err := json.Marshal(&tagMirrorRecord{})
 	require.NoError(t, err)
 	kv := seedKV(t, "tagmirror-mirror-test", map[string][]byte{
-		kvutil.AccountKey("acct", "vol-1"): withTags,
-		kvutil.AccountKey("acct", "vol-2"): nilTags,
+		kvstore.AccountKey("acct", "vol-1"): withTags,
+		kvstore.AccountKey("acct", "vol-2"): nilTags,
 	})
 
 	tagsOf := func(r *tagMirrorRecord) *map[string]string { return &r.Tags }
 
 	readTags := func(t *testing.T, id string) map[string]string {
-		entry, err := kv.Get(t.Context(), kvutil.AccountKey("acct", id))
+		entry, err := kv.Get(t.Context(), kvstore.AccountKey("acct", id))
 		require.NoError(t, err)
 		var got tagMirrorRecord
 		require.NoError(t, json.Unmarshal(entry.Value(), &got))
@@ -150,7 +150,7 @@ func TestMirrorKVRecordTags(t *testing.T) {
 	// must never be touched even though it exists in this bucket.
 	otherRec, err := json.Marshal(&tagMirrorRecord{Tags: map[string]string{"x": "y"}})
 	require.NoError(t, err)
-	_, err = kv.Put(t.Context(), kvutil.AccountKey("acct", "snap-other"), otherRec)
+	_, err = kv.Put(t.Context(), kvstore.AccountKey("acct", "snap-other"), otherRec)
 	require.NoError(t, err)
 	require.NoError(t, MirrorKVRecordTags(t.Context(), kv, "acct", "vol-", resources, tagsOf,
 		MergeTagsMut(&ec2.CreateTagsInput{Tags: []*ec2.Tag{

spinifex/domains/ec2/tags/service_impl.go

@@ -23,7 +23,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -274,7 +274,7 @@ func (s *TagsServiceImpl) CreateTags(ctx context.Context, input *ec2.CreateTagsI
 func (s *TagsServiceImpl) taggedResourceIDs(ctx context.Context, accountID string) ([]string, error) {
 	seen := make(map[string]struct{})
 
-	keys, err := kvutil.Keys(ctx, s.kv)
+	keys, err := kvstore.Keys(ctx, s.kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, fmt.Errorf("list tag keys: %w", err)
 	}

spinifex/domains/ec2/tags/store.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -46,7 +46,7 @@ func resourceIDFromKVKey(key, accountID string) (string, bool) {
 // GetOrCreateTagsBucket opens the tag bucket, creating it at the cluster's
 // replica count if this is the first node to reach it.
 func GetOrCreateTagsBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) {
-	return kvutil.GetOrCreateBucket(ctx, js, KVBucketTags, kvTagsHistory)
+	return kvstore.GetOrCreateBucket(ctx, js, KVBucketTags, kvTagsHistory)
 }
 
 // readKVTags returns a resource's tags from the bucket, and whether the key
@@ -76,7 +76,7 @@ func (s *TagsServiceImpl) readKVTags(ctx context.Context, accountID, resourceID
 // before the store moved.
 func (s *TagsServiceImpl) mutateTags(ctx context.Context, accountID, resourceID string, mutate func(map[string]string)) error {
 	key := tagsKVKey(accountID, resourceID)
-	_, err := kvutil.Update(ctx, s.kv, key, kvutil.CASConfig{
+	_, err := kvstore.Update(ctx, s.kv, key, kvstore.CASConfig{
 		Attempts:       tagCASAttempts,
 		CreateIfAbsent: true,
 		Exhausted: func(key string, attempts int) error {
@@ -104,7 +104,7 @@ func (s *TagsServiceImpl) PutResourceTags(ctx context.Context, accountID, resour
 	if err != nil {
 		return err
 	}
-	return kvutil.Put(ctx, s.kv, tagsKVKey(accountID, resourceID), kvutil.CASConfig{
+	return kvstore.Put(ctx, s.kv, tagsKVKey(accountID, resourceID), kvstore.CASConfig{
 		Attempts:       tagCASAttempts,
 		CreateIfAbsent: true,
 	}, data)

spinifex/domains/ec2/volume/volume_leak_gate_test.go

@@ -0,0 +1,136 @@
+package volume
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"sync"
+	"syscall"
+	"testing"
+
+	"github.com/mulgadc/spinifex/internal/testkit/deadpid"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const leakGateNode = "node-home"
+
+// failedKill is the record a failed QEMU kill leaves: QEMU and volumes failed.
+func failedKill(id, node string) *vm.VM {
+	return &vm.VM{
+		ID: id, LastNode: node, Status: vm.StateTerminated,
+		Teardown: map[string]string{vm.TeardownQEMU: string(vm.TeardownFailed), vm.TeardownVolumes: string(vm.TeardownFailed)},
+	}
+}
+
+// sweepAfterFailedKill runs the real leak reaper over a seeded in-use volume
+// using the leaked set the daemon derives from the terminated record.
+func sweepAfterFailedKill(t *testing.T, rec *vm.VM) (state, attached string, orphaned bool) {
+	t.Helper()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", objectstore.NewMemoryObjectStore())
+	seedVolume(t, svc, "vol-leakgate", "in-use", rec.ID)
+	reaper := svc.NewVolumeLeakReaper(func() (map[string]bool, error) {
+		return vm.LeakedVolumeInstances([]*vm.VM{rec}, leakGateNode), nil
+	})
+	_, err := reaper.Sweep(context.Background())
+	require.NoError(t, err)
+	meta, err := svc.GetVolumeMetadata(testVolAccountID, "vol-leakgate")
+	require.NoError(t, err)
+	return meta.State, meta.AttachedInstance, meta.Tags[orphanTagKey] != ""
+}
+
+func isolatedPidDir(t *testing.T) {
+	t.Helper()
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+}
+
+// stamped is the failed-kill record with QEMU teardown set to state.
+func stamped(state vm.TeardownState) *vm.VM {
+	rec := failedKill("i-leakgate", leakGateNode)
+	rec.Teardown[vm.TeardownQEMU] = string(state)
+	return rec
+}
+
+// TestS7_LeakReaperNeverFreesAVolumeWhileQEMUMayBeAlive enforces ADR-0003:S7: a
+// failed kill leaves the volume in-use and attached until QEMU is proven gone.
+func TestS7_LeakReaperNeverFreesAVolumeWhileQEMUMayBeAlive(t *testing.T) {
+	for _, stamp := range []vm.TeardownState{vm.TeardownFailed, vm.TeardownDone} {
+		t.Run("qemu teardown "+string(stamp), func(t *testing.T) { leakGateKeepsVolume(t, stamp) })
+	}
+}
+
+func leakGateKeepsVolume(t *testing.T, stamp vm.TeardownState) {
+	for _, content := range []string{"not-a-pid", "0", "-1"} {
+		t.Run("pid file "+content, func(t *testing.T) {
+			isolatedPidDir(t)
+			path, err := hostprocess.GeneratePidFile("i-leakgate")
+			require.NoError(t, err)
+			require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
+			state, attached, orphaned := sweepAfterFailedKill(t, stamped(stamp))
+			assert.Equal(t, "in-use", state)
+			assert.Equal(t, "i-leakgate", attached)
+			assert.False(t, orphaned)
+		})
+	}
+
+	t.Run("live process", func(t *testing.T) {
+		isolatedPidDir(t)
+		cmd := exec.Command("sleep", "60")
+		require.NoError(t, cmd.Start())
+		var wg sync.WaitGroup
+		wg.Go(func() { _ = cmd.Wait() })
+		t.Cleanup(func() { _ = cmd.Process.Kill(); wg.Wait() })
+		require.NoError(t, hostprocess.WritePidFile("i-leakgate", cmd.Process.Pid))
+		state, attached, orphaned := sweepAfterFailedKill(t, stamped(stamp))
+		assert.Equal(t, "in-use", state)
+		assert.Equal(t, "i-leakgate", attached)
+		assert.False(t, orphaned)
+	})
+
+	t.Run("unsignalable process", func(t *testing.T) {
+		isolatedPidDir(t)
+		t.Cleanup(hostprocess.SetSignalProbe(func(int) error { return syscall.EPERM }))
+		require.NoError(t, hostprocess.WritePidFile("i-leakgate", deadpid.PID))
+		state, attached, orphaned := sweepAfterFailedKill(t, stamped(stamp))
+		assert.Equal(t, "in-use", state)
+		assert.Equal(t, "i-leakgate", attached)
+		assert.False(t, orphaned)
+	})
+}
+
+// TestLeakReaperFreesTheVolumeOnceQEMUIsProvenGone keeps the existing outcome
+// for each proof of exit: the volume is tagged orphaned and made available.
+func TestLeakReaperFreesTheVolumeOnceQEMUIsProvenGone(t *testing.T) {
+	t.Run("dead process", func(t *testing.T) {
+		isolatedPidDir(t)
+		require.NoError(t, hostprocess.WritePidFile("i-leakgate", deadpid.PID))
+		state, attached, orphaned := sweepAfterFailedKill(t, failedKill("i-leakgate", leakGateNode))
+		assert.Equal(t, "available", state)
+		assert.Empty(t, attached)
+		assert.True(t, orphaned)
+	})
+	t.Run("absent pid file", func(t *testing.T) {
+		isolatedPidDir(t)
+		state, _, orphaned := sweepAfterFailedKill(t, failedKill("i-leakgate", leakGateNode))
+		assert.Equal(t, "available", state)
+		assert.True(t, orphaned)
+	})
+}
+
+// TestS7_LeakedSetJudgesOnlyTheHomeNode enforces ADR-0003:S7: another node's
+// record is never in the set and its PID file is never probed. A counting probe
+// over a present PID file shows the home-node check runs before the QEMU one.
+func TestS7_LeakedSetJudgesOnlyTheHomeNode(t *testing.T) {
+	isolatedPidDir(t)
+	var probes int
+	t.Cleanup(hostprocess.SetSignalProbe(func(int) error { probes++; return syscall.ESRCH }))
+	require.NoError(t, hostprocess.WritePidFile("i-other", deadpid.PID))
+	require.NoError(t, hostprocess.WritePidFile("i-empty", deadpid.PID))
+
+	leaked := vm.LeakedVolumeInstances([]*vm.VM{failedKill("i-other", "node-other"), failedKill("i-empty", "")}, leakGateNode)
+	assert.Empty(t, leaked)
+	assert.Zero(t, probes, "a PID file was judged on a node that does not own the instance")
+}

spinifex/domains/ec2/vpc/default_vpc.go

@@ -12,7 +12,7 @@ import (
 
 	networkv1 "github.com/mulgadc/spinifex/contracts/network/v1"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -105,7 +105,7 @@ func (s *VPCServiceImpl) ensureDefaultVPC(ctx context.Context, accountID string,
 			return claim.info(), nil
 		}
 
-		if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, claim.VpcId)); err == nil {
+		if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, claim.VpcId)); err == nil {
 			return claim.info(), nil
 		} else if !errors.Is(err, jetstream.ErrKeyNotFound) {
 			return nil, fmt.Errorf("read default VPC %s: %w", claim.VpcId, err)
@@ -233,7 +233,7 @@ func oldestAccountRecord[T any](ctx context.Context, kv jetstream.KeyValue, acco
 	var bestID string
 	var bestCreated time.Time
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := kv.Get(ctx, key)
@@ -262,7 +262,7 @@ func oldestAccountRecord[T any](ctx context.Context, kv jetstream.KeyValue, acco
 // marks the claim complete. Records are written with Create, so only the caller
 // that actually wrote one announces it to vpcd.
 func (s *VPCServiceImpl) buildDefaultVPC(ctx context.Context, accountID string, claim *defaultVPCClaim) error {
-	created, err := createRecord(ctx, s.vpcKV, kvutil.AccountKey(accountID, claim.VpcId), VPCRecord{
+	created, err := createRecord(ctx, s.vpcKV, kvstore.AccountKey(accountID, claim.VpcId), VPCRecord{
 		VpcId:              claim.VpcId,
 		CidrBlock:          claim.Cidr,
 		State:              "available",
@@ -284,7 +284,7 @@ func (s *VPCServiceImpl) buildDefaultVPC(ctx context.Context, accountID string,
 	if s.config != nil && s.config.AZ != "" {
 		az = s.config.AZ
 	}
-	created, err = createRecord(ctx, s.subnetKV, kvutil.AccountKey(accountID, claim.SubnetId), SubnetRecord{
+	created, err = createRecord(ctx, s.subnetKV, kvstore.AccountKey(accountID, claim.SubnetId), SubnetRecord{
 		SubnetId:            claim.SubnetId,
 		VpcId:               claim.VpcId,
 		CidrBlock:           claim.SubnetCidr,
@@ -320,7 +320,7 @@ func (s *VPCServiceImpl) buildDefaultVPC(ctx context.Context, accountID string,
 		}
 	}
 
-	if _, err := kvutil.Update(ctx, s.defaultsKV, accountID, kvutil.CASConfig{}, func(c *defaultVPCClaim) (bool, error) {
+	if _, err := kvstore.Update(ctx, s.defaultsKV, accountID, kvstore.CASConfig{}, func(c *defaultVPCClaim) (bool, error) {
 		if c.Complete || c.VpcId != claim.VpcId {
 			return false, nil
 		}

spinifex/domains/ec2/vpc/default_vpc_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
@@ -149,12 +149,12 @@ func TestEnsureDefaultVPC_AdoptsOldestExistingDefault(t *testing.T) {
 	const accountID = "000000000051"
 	created := time.Date(2026, 9, 11, 14, 13, 40, 0, time.UTC)
 	for i, id := range []string{"vpc-0000000000000newer", "vpc-0000000000000older"} {
-		putRecord(t, js, ec2vpc.KVBucketVPCs, kvutil.AccountKey(accountID, id), ec2vpc.VPCRecord{
+		putRecord(t, js, ec2vpc.KVBucketVPCs, kvstore.AccountKey(accountID, id), ec2vpc.VPCRecord{
 			VpcId: id, CidrBlock: ec2vpc.DefaultVPCCidr, IsDefault: true, VNI: int64(200 + i),
 			CreatedAt: created.Add(time.Duration(1-i) * time.Minute),
 		})
 	}
-	putRecord(t, js, ec2vpc.KVBucketSubnets, kvutil.AccountKey(accountID, "subnet-00000000000older"), ec2vpc.SubnetRecord{
+	putRecord(t, js, ec2vpc.KVBucketSubnets, kvstore.AccountKey(accountID, "subnet-00000000000older"), ec2vpc.SubnetRecord{
 		SubnetId: "subnet-00000000000older", VpcId: "vpc-0000000000000older", CidrBlock: ec2vpc.DefaultSubnetCidr,
 		IsDefault: true, CreatedAt: created,
 	})
@@ -183,7 +183,7 @@ func TestEnsureDefaultVPC_FinishesHalfBuiltClaim(t *testing.T) {
 		"route_table_id": "rtb-00000000000half", "internet_gateway_id": "igw-00000000000half",
 		"cidr": ec2vpc.DefaultVPCCidr, "subnet_cidr": ec2vpc.DefaultSubnetCidr, "vni": 4242,
 	})
-	putRecord(t, js, ec2vpc.KVBucketVPCs, kvutil.AccountKey(accountID, "vpc-00000000000000half"), ec2vpc.VPCRecord{
+	putRecord(t, js, ec2vpc.KVBucketVPCs, kvstore.AccountKey(accountID, "vpc-00000000000000half"), ec2vpc.VPCRecord{
 		VpcId: "vpc-00000000000000half", CidrBlock: ec2vpc.DefaultVPCCidr, IsDefault: true, VNI: 4242,
 	})
 

spinifex/domains/ec2/vpc/eni.go

@@ -23,7 +23,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -118,7 +118,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2.
 	subnetId := *input.SubnetId
 
 	// Verify subnet exists and belongs to this account
-	subnetEntry, err := s.subnetKV.Get(ctx, kvutil.AccountKey(accountID, subnetId))
+	subnetEntry, err := s.subnetKV.Get(ctx, kvstore.AccountKey(accountID, subnetId))
 	if err != nil {
 		return nil, errors.New(awserrors.ErrorInvalidSubnetIDNotFound)
 	}
@@ -205,7 +205,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2.
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal ENI record: %w", err)
 	}
-	if _, err := s.eniKV.Put(ctx, kvutil.AccountKey(accountID, eniId), data); err != nil {
+	if _, err := s.eniKV.Put(ctx, kvstore.AccountKey(accountID, eniId), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -251,7 +251,7 @@ func (s *VPCServiceImpl) ForceDeleteInstanceENI(ctx context.Context, accountID,
 }
 
 func (s *VPCServiceImpl) deleteNetworkInterface(ctx context.Context, eniId, accountID string, force bool) (*ec2.DeleteNetworkInterfaceOutput, error) {
-	key := kvutil.AccountKey(accountID, eniId)
+	key := kvstore.AccountKey(accountID, eniId)
 
 	// Get the ENI record
 	entry, err := s.eniKV.Get(ctx, key)
@@ -346,7 +346,7 @@ func (s *VPCServiceImpl) releaseENISideEffects(ctx context.Context, eniId, accou
 // deletion that did not happen.
 func (s *VPCServiceImpl) DetachAndDeleteENI(ctx context.Context, accountID, eniID string, force bool) (deleted bool, err error) {
 	const maxAttempts = 3
-	key := kvutil.AccountKey(accountID, eniID)
+	key := kvstore.AccountKey(accountID, eniID)
 	var lastErr error
 
 	for range maxAttempts {
@@ -458,7 +458,7 @@ func (s *VPCServiceImpl) ModifyNetworkInterfaceAttribute(ctx context.Context, in
 	}
 
 	eniId := *input.NetworkInterfaceId
-	key := kvutil.AccountKey(accountID, eniId)
+	key := kvstore.AccountKey(accountID, eniId)
 
 	entry, err := s.eniKV.Get(ctx, key)
 	if err != nil {
@@ -569,7 +569,7 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *e
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -707,7 +707,7 @@ func (s *VPCServiceImpl) AttachENIWithOwner(accountID, eniId, instanceId string,
 
 func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instanceId string, deviceIndex int64, instanceOwnerID string) (string, error) {
 	attachmentId := awsidentifiers.GenerateResourceID("eni-attach")
-	err := s.casENI(ctx, kvutil.AccountKey(accountID, eniId), eniId, func(record *ENIRecord) error {
+	err := s.casENI(ctx, kvstore.AccountKey(accountID, eniId), eniId, func(record *ENIRecord) error {
 		if record.Status == "in-use" {
 			return errors.New(awserrors.ErrorInvalidNetworkInterfaceInUse)
 		}
@@ -737,7 +737,7 @@ func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instan
 // DetachENI marks an ENI as detached from an instance (internal use by TerminateInstances).
 // accountID scopes the lookup to the correct KV key.
 func (s *VPCServiceImpl) DetachENI(ctx context.Context, accountID, eniId string) error {
-	key := kvutil.AccountKey(accountID, eniId)
+	key := kvstore.AccountKey(accountID, eniId)
 	entry, err := s.eniKV.Get(ctx, key)
 	if err != nil {
 		return eniNotFoundError(eniId)
@@ -773,7 +773,7 @@ func (s *VPCServiceImpl) GetENIRecord(accountID, eniId string) (ENIRecord, error
 }
 
 func (s *VPCServiceImpl) getENIRecord(ctx context.Context, accountID, eniId string) (ENIRecord, error) {
-	key := kvutil.AccountKey(accountID, eniId)
+	key := kvstore.AccountKey(accountID, eniId)
 	entry, err := s.eniKV.Get(ctx, key)
 	if err != nil {
 		return ENIRecord{}, eniNotFoundError(eniId)
@@ -793,7 +793,7 @@ func (s *VPCServiceImpl) UpdateENI(accountID, eniId string, fn func(*ENIRecord))
 }
 
 func (s *VPCServiceImpl) updateENI(ctx context.Context, accountID, eniId string, fn func(*ENIRecord)) error {
-	return s.casENI(ctx, kvutil.AccountKey(accountID, eniId), eniId, func(r *ENIRecord) error {
+	return s.casENI(ctx, kvstore.AccountKey(accountID, eniId), eniId, func(r *ENIRecord) error {
 		fn(r)
 		return nil
 	})
@@ -876,7 +876,7 @@ func (s *VPCServiceImpl) listInstanceENIs(ctx context.Context, accountID, instan
 	prefix := accountID + "."
 	var out []ENIRecord
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, key)
@@ -938,7 +938,7 @@ func (s *VPCServiceImpl) scanENIs(ctx context.Context, match func(*ENIRecord) bo
 
 	var out []AccountENI
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		accountID, _, found := strings.Cut(key, ".")
@@ -1004,7 +1004,7 @@ func (s *VPCServiceImpl) findENIByAttachment(ctx context.Context, accountID, att
 	}
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, key)
@@ -1028,7 +1028,7 @@ func (s *VPCServiceImpl) UpdateENIPublicIP(accountID, eniId, publicIP, poolName
 }
 
 func (s *VPCServiceImpl) updateENIPublicIP(ctx context.Context, accountID, eniId, publicIP, poolName string) error {
-	key := kvutil.AccountKey(accountID, eniId)
+	key := kvstore.AccountKey(accountID, eniId)
 	entry, err := s.eniKV.Get(ctx, key)
 	if err != nil {
 		return fmt.Errorf("ENI %s not found: %w", eniId, err)
@@ -1063,7 +1063,7 @@ func (s *VPCServiceImpl) fillSGNames(ctx context.Context, accountID string, ids
 			continue
 		}
 		names[id] = ""
-		entry, err := s.sgKV.Get(ctx, kvutil.AccountKey(accountID, id))
+		entry, err := s.sgKV.Get(ctx, kvstore.AccountKey(accountID, id))
 		if err != nil {
 			slog.WarnContext(ctx, "fillSGNames: SG read failed", "groupId", id, "err", err)
 			continue
@@ -1080,7 +1080,7 @@ func (s *VPCServiceImpl) fillSGNames(ctx context.Context, accountID string, ids
 // vpcDNSHostnames reports whether the VPC has enableDnsHostnames on. An
 // unreadable VPC reads as off, so its interfaces go unnamed.
 func (s *VPCServiceImpl) vpcDNSHostnames(ctx context.Context, accountID, vpcID string) bool {
-	entry, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID))
+	entry, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID))
 	if err != nil {
 		slog.WarnContext(ctx, "vpcDNSHostnames: VPC read failed", "vpcId", vpcID, "err", err)
 		return false
@@ -1238,7 +1238,7 @@ func (s *VPCServiceImpl) validateSGAttachment(ctx context.Context, accountID str
 
 	// Each SG must exist in the caller's account and belong to the same VPC.
 	for _, sgId := range sgIds {
-		sgEntry, err := s.sgKV.Get(ctx, kvutil.AccountKey(accountID, sgId))
+		sgEntry, err := s.sgKV.Get(ctx, kvstore.AccountKey(accountID, sgId))
 		if err != nil {
 			return sgNotFoundError(sgId)
 		}

spinifex/domains/ec2/vpc/eni_lease_rebind.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/domains/network/projection"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/topology"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -69,7 +69,7 @@ func (s *VPCServiceImpl) findENIAnyAccount(ctx context.Context, eniID string) (*
 
 	suffix := "." + eniID
 	for _, k := range keys {
-		if k == kvutil.VersionKey || !strings.HasSuffix(k, suffix) {
+		if k == kvstore.VersionKey || !strings.HasSuffix(k, suffix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, k)

spinifex/domains/ec2/vpc/eni_stale_read_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -30,7 +30,7 @@ func newStaleReadENI(t *testing.T) (*VPCServiceImpl, string, func() ENIRecord) {
 	// Reads the stored record directly: the wrapped bucket would serve the
 	// check itself from a lagging replica.
 	stored := func() ENIRecord {
-		entry, err := inner.Get(context.Background(), kvutil.AccountKey(testAccountID, eniID))
+		entry, err := inner.Get(context.Background(), kvstore.AccountKey(testAccountID, eniID))
 		require.NoError(t, err)
 		var rec ENIRecord
 		require.NoError(t, json.Unmarshal(entry.Value(), &rec))
@@ -114,7 +114,7 @@ func TestCASENIGivesUpAfterBoundedConflicts(t *testing.T) {
 	t.Parallel()
 	svc, eniID, _ := newStaleReadENI(t)
 	conflicts := 0
-	err := svc.casENI(t.Context(), kvutil.AccountKey(testAccountID, eniID), eniID, func(*ENIRecord) error {
+	err := svc.casENI(t.Context(), kvstore.AccountKey(testAccountID, eniID), eniID, func(*ENIRecord) error {
 		conflicts++
 		// Move the record under the next compare-and-swap, as a competing writer does.
 		return svc.UpdateENI(testAccountID, eniID, func(r *ENIRecord) { r.Description = "other writer" })

spinifex/domains/ec2/vpc/lease_owner.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -42,7 +42,7 @@ func keyWithSuffixExists(ctx context.Context, kv jetstream.KeyValue, resourceID
 	}
 	suffix := "." + resourceID
 	for _, k := range keys {
-		if k != kvutil.VersionKey && strings.HasSuffix(k, suffix) {
+		if k != kvstore.VersionKey && strings.HasSuffix(k, suffix) {
 			return true, nil
 		}
 	}

spinifex/domains/ec2/vpc/security_group.go

@@ -23,7 +23,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -245,7 +245,7 @@ func (s *VPCServiceImpl) CreateSecurityGroup(ctx context.Context, input *ec2.Cre
 	}
 	sgsInVPC := 0
 	for _, k := range sgKeys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -305,7 +305,7 @@ func (s *VPCServiceImpl) CreateSecurityGroup(ctx context.Context, input *ec2.Cre
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal security group record: %w", err)
 	}
-	if _, err := s.sgKV.Put(ctx, kvutil.AccountKey(accountID, groupId), data); err != nil {
+	if _, err := s.sgKV.Put(ctx, kvstore.AccountKey(accountID, groupId), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -331,7 +331,7 @@ func (s *VPCServiceImpl) DeleteSecurityGroup(ctx context.Context, input *ec2.Del
 	}
 
 	groupId := *input.GroupId
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -380,7 +380,7 @@ func (s *VPCServiceImpl) validateSGRuleReferences(ctx context.Context, accountID
 		if r.SourceSG == "" {
 			continue
 		}
-		entry, err := s.sgKV.Get(ctx, kvutil.AccountKey(accountID, r.SourceSG))
+		entry, err := s.sgKV.Get(ctx, kvstore.AccountKey(accountID, r.SourceSG))
 		if err != nil {
 			return sgNotFoundError(r.SourceSG)
 		}
@@ -406,7 +406,7 @@ func (s *VPCServiceImpl) checkSGDependencies(ctx context.Context, accountID, gro
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range eniKeys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, k)
@@ -435,7 +435,7 @@ func (s *VPCServiceImpl) checkSGDependencies(ctx context.Context, accountID, gro
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range sgKeys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, k)
@@ -561,7 +561,7 @@ func (s *VPCServiceImpl) DescribeSecurityGroups(ctx context.Context, input *ec2.
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -713,7 +713,7 @@ func (s *VPCServiceImpl) GetSecurityGroupsForVpc(ctx context.Context, input *ec2
 
 	groups := []*ec2.SecurityGroupForVpc{}
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, key)
@@ -844,7 +844,7 @@ func (s *VPCServiceImpl) DescribeSecurityGroupRules(ctx context.Context, input *
 	emitted := make(map[string]bool)
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, key)
@@ -945,7 +945,7 @@ func (s *VPCServiceImpl) updateSGRuleTags(ctx context.Context, accountID, ruleID
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, key)
@@ -1056,7 +1056,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupIngress(ctx context.Context, inpu
 	}
 
 	groupId := *input.GroupId
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -1134,7 +1134,7 @@ func (s *VPCServiceImpl) AuthorizeSecurityGroupEgress(ctx context.Context, input
 	}
 
 	groupId := *input.GroupId
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -1212,7 +1212,7 @@ func (s *VPCServiceImpl) RevokeSecurityGroupIngress(ctx context.Context, input *
 	}
 
 	groupId := *input.GroupId
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -1277,7 +1277,7 @@ func (s *VPCServiceImpl) RevokeSecurityGroupEgress(ctx context.Context, input *e
 	}
 
 	groupId := *input.GroupId
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -1397,7 +1397,7 @@ func (s *VPCServiceImpl) updateSGRuleDescriptions(ctx context.Context, accountID
 		return errors.New(awserrors.ErrorInvalidParameterCombination)
 	}
 
-	key := kvutil.AccountKey(accountID, req.groupId)
+	key := kvstore.AccountKey(accountID, req.groupId)
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
 		return sgNotFoundError(req.groupId)
@@ -1581,7 +1581,7 @@ func (s *VPCServiceImpl) ModifySecurityGroupRules(ctx context.Context, input *ec
 	}
 
 	groupId := *input.GroupId
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
@@ -2158,7 +2158,7 @@ func (s *VPCServiceImpl) storeDefaultSecurityGroup(ctx context.Context, accountI
 	if err != nil {
 		return nil, false, fmt.Errorf("marshal default security group: %w", err)
 	}
-	if _, err := s.sgKV.Create(ctx, kvutil.AccountKey(accountID, groupId), data); err != nil {
+	if _, err := s.sgKV.Create(ctx, kvstore.AccountKey(accountID, groupId), data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
 			return record, false, nil
 		}
@@ -2182,7 +2182,7 @@ func (s *VPCServiceImpl) announceDefaultSecurityGroup(record *SecurityGroupRecor
 // CannotDelete guard for default SGs. Used by DeleteVpc to cascade-delete the
 // per-VPC default SG. Surfaces vpcd tear-down failures to the caller.
 func (s *VPCServiceImpl) deleteSecurityGroupInternal(ctx context.Context, accountID, groupId string) error {
-	key := kvutil.AccountKey(accountID, groupId)
+	key := kvstore.AccountKey(accountID, groupId)
 	entry, err := s.sgKV.Get(ctx, key)
 	if err != nil {
 		return fmt.Errorf("read default security group: %w", err)
@@ -2219,7 +2219,7 @@ func (s *VPCServiceImpl) findDefaultSGForVPC(ctx context.Context, accountID, vpc
 		return "", err
 	}
 	for _, k := range keys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, k)

spinifex/domains/ec2/vpc/security_group_modify_test.go

@@ -15,7 +15,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -62,7 +62,7 @@ func tcpPeerRequest(port int64, peer string) *ec2.SecurityGroupRuleRequest {
 // modify wrote nothing.
 func storedSGRecord(t *testing.T, svc *VPCServiceImpl, sgID string) []byte {
 	t.Helper()
-	entry, err := svc.sgKV.Get(t.Context(), kvutil.AccountKey(testAccountID, sgID))
+	entry, err := svc.sgKV.Get(t.Context(), kvstore.AccountKey(testAccountID, sgID))
 	require.NoError(t, err)
 	return entry.Value()
 }
@@ -689,7 +689,7 @@ func TestRevokeSecurityGroupIngress_InvertedPortRangeStillRevocable(t *testing.T
 	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
 	sgID := createTestSG(t, svc, vpcID, "stale-inverted-ports")
 
-	key := kvutil.AccountKey(testAccountID, sgID)
+	key := kvstore.AccountKey(testAccountID, sgID)
 	var rec SecurityGroupRecord
 	require.NoError(t, json.Unmarshal(storedSGRecord(t, svc, sgID), &rec))
 	rec.IngressRules = append(rec.IngressRules, SGRule{

spinifex/domains/ec2/vpc/security_group_rule_validation_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -35,7 +35,7 @@ func plantIngressRules(t *testing.T, svc *VPCServiceImpl, sgID string, rules ...
 	rec.IngressRules = append(rec.IngressRules, rules...)
 	data, err := json.Marshal(rec)
 	require.NoError(t, err)
-	_, err = svc.sgKV.Put(t.Context(), kvutil.AccountKey(testAccountID, sgID), data)
+	_, err = svc.sgKV.Put(t.Context(), kvstore.AccountKey(testAccountID, sgID), data)
 	require.NoError(t, err)
 }
 

spinifex/domains/ec2/vpc/security_group_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -1524,7 +1524,7 @@ func findDefaultSGInVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) string
 	require.NoError(t, err)
 	var matches []string
 	for _, k := range keys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		entry, err := svc.sgKV.Get(t.Context(), k)
@@ -2122,7 +2122,7 @@ func seedAvailableVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) {
 	rec := VPCRecord{VpcId: vpcID, CidrBlock: "10.0.0.0/16", State: "available"}
 	data, err := json.Marshal(rec)
 	require.NoError(t, err)
-	_, err = svc.vpcKV.Put(t.Context(), kvutil.AccountKey(testAccountID, vpcID), data)
+	_, err = svc.vpcKV.Put(t.Context(), kvstore.AccountKey(testAccountID, vpcID), data)
 	require.NoError(t, err)
 }
 
@@ -2359,7 +2359,7 @@ func TestAuthorizeSecurityGroupIngress_AssignsRuleIDs(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 
-	entry, err := svc.sgKV.Get(t.Context(), kvutil.AccountKey(testAccountID, sgID))
+	entry, err := svc.sgKV.Get(t.Context(), kvstore.AccountKey(testAccountID, sgID))
 	require.NoError(t, err)
 	var record SecurityGroupRecord
 	require.NoError(t, json.Unmarshal(entry.Value(), &record))

spinifex/domains/ec2/vpc/service_impl.go

@@ -23,7 +23,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -166,7 +166,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("create JetStream client: %w", err)
 	}
 
-	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketVPCs, 10)
+	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketVPCs, err)
 	}
@@ -174,7 +174,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketVPCs, err)
 	}
 
-	subnetKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketSubnets, 10)
+	subnetKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketSubnets, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSubnets, err)
 	}
@@ -182,7 +182,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketSubnets, err)
 	}
 
-	vniKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketVNICounter, 10)
+	vniKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketVNICounter, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketVNICounter, err)
 	}
@@ -190,7 +190,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketVNICounter, err)
 	}
 
-	eniKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketENIs, 10)
+	eniKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketENIs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketENIs, err)
 	}
@@ -198,7 +198,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketENIs, err)
 	}
 
-	sgKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketSecurityGroups, 10)
+	sgKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketSecurityGroups, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketSecurityGroups, err)
 	}
@@ -206,7 +206,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketSecurityGroups, err)
 	}
 
-	defaultsKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketDefaultVPCs, 10)
+	defaultsKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketDefaultVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketDefaultVPCs, err)
 	}
@@ -214,18 +214,18 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("migrate %s: %w", KVBucketDefaultVPCs, err)
 	}
 
-	rtbKV, err := kvutil.GetOrCreateBucket(ctx, js, "spinifex-vpc-route-tables", 10)
+	rtbKV, err := kvstore.GetOrCreateBucket(ctx, js, "spinifex-vpc-route-tables", 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket spinifex-vpc-route-tables: %w", err)
 	}
 
 	// Named literally, like the route table bucket above: the IGW package
 	// imports this one, so its constant cannot be referenced here.
-	igwKV, err := kvutil.GetOrCreateBucket(ctx, js, "spinifex-igw", 10)
+	igwKV, err := kvstore.GetOrCreateBucket(ctx, js, "spinifex-igw", 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket spinifex-igw: %w", err)
 	}
-	eigwKV, err := kvutil.GetOrCreateBucket(ctx, js, "spinifex-eigw", 10)
+	eigwKV, err := kvstore.GetOrCreateBucket(ctx, js, "spinifex-eigw", 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket spinifex-eigw: %w", err)
 	}
@@ -261,7 +261,7 @@ func NewVPCServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 // nextVNI allocates the next VNI using atomic increment on the NATS KV counter.
 func (s *VPCServiceImpl) nextVNI(ctx context.Context) (int64, error) {
 	var allocated int64
-	_, err := kvutil.Update(ctx, s.vniKV, vniCounterKey, kvutil.CASConfig{CreateIfAbsent: true, Attempts: vniCASAttempts},
+	_, err := kvstore.Update(ctx, s.vniKV, vniCounterKey, kvstore.CASConfig{CreateIfAbsent: true, Attempts: vniCASAttempts},
 		func(next *int64) (bool, error) {
 			if *next == 0 {
 				*next = vniStart
@@ -324,7 +324,7 @@ func (s *VPCServiceImpl) CreateVpc(ctx context.Context, input *ec2.CreateVpcInpu
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal VPC record: %w", err)
 	}
-	if _, err := s.vpcKV.Put(ctx, kvutil.AccountKey(accountID, vpcID), data); err != nil {
+	if _, err := s.vpcKV.Put(ctx, kvstore.AccountKey(accountID, vpcID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -359,7 +359,7 @@ func (s *VPCServiceImpl) CreateVpc(ctx context.Context, input *ec2.CreateVpcInpu
 // requireVPCExists returns InvalidVpcID.NotFound if the VPC doesn't exist for
 // this account.
 func (s *VPCServiceImpl) requireVPCExists(ctx context.Context, accountID, vpcId string) error {
-	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcId)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcId)); err != nil {
 		return vpcNotFoundError(vpcId)
 	}
 	return nil
@@ -372,7 +372,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 	}
 
 	vpcID := *input.VpcId
-	key := kvutil.AccountKey(accountID, vpcID)
+	key := kvstore.AccountKey(accountID, vpcID)
 
 	if _, err := s.vpcKV.Get(ctx, key); err != nil {
 		// AWS-faithful: an absent VPC is NotFound (the tofu/SDK provider
@@ -391,7 +391,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range subnetKeys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -428,7 +428,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range sgKeys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.sgKV.Get(ctx, k)
@@ -465,7 +465,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, k := range rtbKeys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.rtbKV.Get(ctx, k)
@@ -526,7 +526,7 @@ func (s *VPCServiceImpl) DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInpu
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	if rtbID != "" {
-		if err := s.rtbKV.Delete(ctx, kvutil.AccountKey(accountID, rtbID)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
+		if err := s.rtbKV.Delete(ctx, kvstore.AccountKey(accountID, rtbID)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
 			slog.ErrorContext(ctx, "DeleteVpc: main route table reap failed", "vpcId", vpcID, "routeTableId", rtbID, "err", err)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
@@ -567,7 +567,7 @@ func rejectAttachedGateway(ctx context.Context, kv jetstream.KeyValue, accountID
 	}
 
 	for _, k := range keys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := kv.Get(ctx, k)
@@ -646,7 +646,7 @@ func (s *VPCServiceImpl) DescribeVpcs(ctx context.Context, input *ec2.DescribeVp
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -716,7 +716,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
 	}
 
 	// Verify VPC exists and belongs to this account
-	vpcEntry, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID))
+	vpcEntry, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID))
 	if err != nil {
 		return nil, vpcNotFoundError(vpcID)
 	}
@@ -750,7 +750,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
 	}
 
 	for _, k := range subnetKeys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -805,7 +805,7 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal subnet record: %w", err)
 	}
-	if _, err := s.subnetKV.Put(ctx, kvutil.AccountKey(accountID, subnetID), data); err != nil {
+	if _, err := s.subnetKV.Put(ctx, kvstore.AccountKey(accountID, subnetID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -828,7 +828,7 @@ func (s *VPCServiceImpl) DeleteSubnet(ctx context.Context, input *ec2.DeleteSubn
 	}
 
 	subnetID := *input.SubnetId
-	key := kvutil.AccountKey(accountID, subnetID)
+	key := kvstore.AccountKey(accountID, subnetID)
 
 	// Read subnet record before deletion (needed for vpcd event)
 	subnetEntry, err := s.subnetKV.Get(ctx, key)
@@ -887,7 +887,7 @@ func (s *VPCServiceImpl) checkSubnetResidents(ctx context.Context, accountID, su
 	}
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.eniKV.Get(ctx, key)
@@ -935,7 +935,7 @@ func (s *VPCServiceImpl) clearRouteTableAssociationsForSubnet(ctx context.Contex
 	}
 	prefix := accountID + "."
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.rtbKV.Get(ctx, key)
@@ -1020,7 +1020,7 @@ func (s *VPCServiceImpl) DescribeSubnets(ctx context.Context, input *ec2.Describ
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -1351,7 +1351,7 @@ func (s *VPCServiceImpl) ModifySubnetAttribute(ctx context.Context, input *ec2.M
 	}
 
 	subnetID := *input.SubnetId
-	key := kvutil.AccountKey(accountID, subnetID)
+	key := kvstore.AccountKey(accountID, subnetID)
 
 	entry, err := s.subnetKV.Get(ctx, key)
 	if err != nil {
@@ -1403,7 +1403,7 @@ func (s *VPCServiceImpl) ModifyVpcAttribute(ctx context.Context, input *ec2.Modi
 	}
 
 	vpcID := *input.VpcId
-	key := kvutil.AccountKey(accountID, vpcID)
+	key := kvstore.AccountKey(accountID, vpcID)
 
 	entry, err := s.vpcKV.Get(ctx, key)
 	if err != nil {
@@ -1451,7 +1451,7 @@ func (s *VPCServiceImpl) DescribeVpcAttribute(ctx context.Context, input *ec2.De
 	}
 
 	vpcID := *input.VpcId
-	key := kvutil.AccountKey(accountID, vpcID)
+	key := kvstore.AccountKey(accountID, vpcID)
 
 	entry, err := s.vpcKV.Get(ctx, key)
 	if err != nil {
@@ -1552,7 +1552,7 @@ func (s *VPCServiceImpl) writeMainRouteTable(ctx context.Context, accountID, vpc
 	if err != nil {
 		return false, fmt.Errorf("marshal main route table: %w", err)
 	}
-	if _, err := s.rtbKV.Create(ctx, kvutil.AccountKey(accountID, rtbID), data); err != nil {
+	if _, err := s.rtbKV.Create(ctx, kvstore.AccountKey(accountID, rtbID), data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
 			return false, nil
 		}
@@ -1576,7 +1576,7 @@ func (s *VPCServiceImpl) findMainRouteTableID(ctx context.Context, accountID, vp
 		return "", fmt.Errorf("list rtb keys: %w", err)
 	}
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.rtbKV.Get(ctx, key)
@@ -1617,7 +1617,7 @@ func (s *VPCServiceImpl) getDefaultSubnet(ctx context.Context, accountID string)
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -1645,7 +1645,7 @@ func (s *VPCServiceImpl) GetSubnet(accountID, subnetId string) (*SubnetRecord, e
 }
 
 func (s *VPCServiceImpl) getSubnet(ctx context.Context, accountID, subnetId string) (*SubnetRecord, error) {
-	key := kvutil.AccountKey(accountID, subnetId)
+	key := kvstore.AccountKey(accountID, subnetId)
 	entry, err := s.subnetKV.Get(ctx, key)
 	if err != nil {
 		return nil, fmt.Errorf("subnet %s not found: %w", subnetId, err)

spinifex/domains/ecr/auth/ecrauth_test.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/golang-jwt/jwt/v5"
 	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -111,7 +111,7 @@ func TestLoadOrCreateSigningKey_FailsWhenNoKeyDecrypts(t *testing.T) {
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "master key does not match")
 
-	entries, err := kvutil.Keys(t.Context(), kv)
+	entries, err := kvstore.Keys(t.Context(), kv)
 	require.NoError(t, err)
 	assert.Len(t, entries, 1, "a failed load must not mint a key alongside the unreadable one")
 }

spinifex/domains/ecr/auth/signingkey.go

@@ -21,7 +21,7 @@ import (
 	"time"
 
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -111,7 +111,7 @@ func openSigningBucket(ctx context.Context, js jetstream.JetStream, masterKey []
 	if len(masterKey) == 0 {
 		return nil, errors.New("ecrauth: empty master key")
 	}
-	return kvutil.GetOrCreateBucket(ctx, js, SigningBucket, signingKeyHistory)
+	return kvstore.GetOrCreateBucket(ctx, js, SigningBucket, signingKeyHistory)
 }
 
 // reloadKeys reads every stored signing key, returning the verify set, each
@@ -125,7 +125,7 @@ func openSigningBucket(ctx context.Context, js jetstream.JetStream, masterKey []
 // load instead takes the gateway down on every node, permanently, over a key no
 // live request needs.
 func reloadKeys(ctx context.Context, kv jetstream.KeyValue, masterKey []byte) (active *SigningKey, verify map[string]*ecdsa.PublicKey, metas []keyMeta, undecodable int, err error) {
-	names, err := kvutil.Keys(ctx, kv)
+	names, err := kvstore.Keys(ctx, kv)
 	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil, nil, nil, 0, fmt.Errorf("list signing keys: %w", err)
 	}

spinifex/domains/ecr/meta.go

@@ -10,7 +10,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -179,7 +179,7 @@ func (s *KVMetaStore) bucket(ctx context.Context, accountID string) (jetstream.K
 		return kv, nil
 	}
 	name := KVAccountBucket(accountID)
-	kv, err := kvutil.GetOrCreateBucket(ctx, s.js, name, KVBucketAccountHistory)
+	kv, err := kvstore.GetOrCreateBucket(ctx, s.js, name, KVBucketAccountHistory)
 	if err != nil {
 		return nil, fmt.Errorf("ecr: open account bucket %s: %w", name, err)
 	}

spinifex/domains/ecs/eni_test.go

@@ -322,7 +322,7 @@ func TestReaper_Awsvpc_ReleasesENI(t *testing.T) {
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
-	sc := NewScheduler(nc, svc, "test-holder")
+	sc := testScheduler(t, nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	assert.Equal(t, 1, eni.releaseCalls)